From fa3dc0037f445a9837099e3d1451cf86d0f40a9e Mon Sep 17 00:00:00 2001 From: Francisco Bernal Date: Thu, 6 Jul 2023 09:12:20 -0700 Subject: [PATCH] feat: Face Mesh Detection (#476) --- .github/workflows/flutter.yml | 4 + README.md | 2 +- packages/example/ios/Podfile.lock | 7 + packages/example/lib/main.dart | 2 + .../face_mesh_detector_view.dart | 84 +++++++++ .../painters/face_mesh_detector_painter.dart | 103 +++++++++++ packages/example/pubspec.lock | 79 +++++++- packages/example/pubspec.yaml | 6 +- .../.gitignore | 29 +++ .../CHANGELOG.md | 3 + .../google_mlkit_face_mesh_detection/LICENSE | 21 +++ .../README.md | 88 +++++++++ .../analysis_options.yaml | 1 + .../android/.gitignore | 8 + .../android/build.gradle | 39 ++++ .../android/settings.gradle | 1 + .../android/src/main/AndroidManifest.xml | 3 + .../FaceMeshDetector.java | 159 +++++++++++++++++ .../GoogleMlKitFaceMeshDetectionPlugin.java | 22 +++ .../example/README.md | 3 + .../ios/.gitignore | 38 ++++ .../ios/Assets/.gitkeep | 0 .../GoogleMlKitFaceMeshDetectionPlugin.h | 4 + .../GoogleMlKitFaceMeshDetectionPlugin.m | 45 +++++ .../google_mlkit_face_mesh_detection.podspec | 27 +++ .../lib/google_mlkit_face_mesh_detection.dart | 3 + .../lib/src/face_mesh_detector.dart | 168 ++++++++++++++++++ .../pubspec.yaml | 28 +++ update_libs.sh | 3 + 29 files changed, 969 insertions(+), 11 deletions(-) create mode 100644 packages/example/lib/vision_detector_views/face_mesh_detector_view.dart create mode 100644 packages/example/lib/vision_detector_views/painters/face_mesh_detector_painter.dart create mode 100644 packages/google_mlkit_face_mesh_detection/.gitignore create mode 100644 packages/google_mlkit_face_mesh_detection/CHANGELOG.md create mode 100644 packages/google_mlkit_face_mesh_detection/LICENSE create mode 100644 packages/google_mlkit_face_mesh_detection/README.md create mode 100644 packages/google_mlkit_face_mesh_detection/analysis_options.yaml create mode 100644 packages/google_mlkit_face_mesh_detection/android/.gitignore create mode 100644 packages/google_mlkit_face_mesh_detection/android/build.gradle create mode 100644 packages/google_mlkit_face_mesh_detection/android/settings.gradle create mode 100644 packages/google_mlkit_face_mesh_detection/android/src/main/AndroidManifest.xml create mode 100644 packages/google_mlkit_face_mesh_detection/android/src/main/java/com/google_mlkit_face_mesh_detection/FaceMeshDetector.java create mode 100644 packages/google_mlkit_face_mesh_detection/android/src/main/java/com/google_mlkit_face_mesh_detection/GoogleMlKitFaceMeshDetectionPlugin.java create mode 100644 packages/google_mlkit_face_mesh_detection/example/README.md create mode 100644 packages/google_mlkit_face_mesh_detection/ios/.gitignore create mode 100644 packages/google_mlkit_face_mesh_detection/ios/Assets/.gitkeep create mode 100644 packages/google_mlkit_face_mesh_detection/ios/Classes/GoogleMlKitFaceMeshDetectionPlugin.h create mode 100644 packages/google_mlkit_face_mesh_detection/ios/Classes/GoogleMlKitFaceMeshDetectionPlugin.m create mode 100644 packages/google_mlkit_face_mesh_detection/ios/google_mlkit_face_mesh_detection.podspec create mode 100644 packages/google_mlkit_face_mesh_detection/lib/google_mlkit_face_mesh_detection.dart create mode 100644 packages/google_mlkit_face_mesh_detection/lib/src/face_mesh_detector.dart create mode 100644 packages/google_mlkit_face_mesh_detection/pubspec.yaml diff --git a/.github/workflows/flutter.yml b/.github/workflows/flutter.yml index c4d7bf05..a9ea4655 100644 --- a/.github/workflows/flutter.yml +++ b/.github/workflows/flutter.yml @@ -37,6 +37,10 @@ jobs: working-directory: ./packages/google_mlkit_face_detection run: flutter pub get + - name: Install dependencies for google_mlkit_face_mesh_detection + working-directory: ./packages/google_mlkit_face_mesh_detection + run: flutter pub get + - name: Install dependencies for google_mlkit_image_labeling working-directory: ./packages/google_mlkit_image_labeling run: flutter pub get diff --git a/README.md b/README.md index ccdf27ea..5fe32650 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Google's ML Kit for Flutter is a set of [Flutter plugins](https://flutter.io/pla |------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------|----| | [Barcode Scanning](https://developers.google.com/ml-kit/vision/barcode-scanning) | [google\_mlkit\_barcode\_scanning](https://pub.dev/packages/google_mlkit_barcode_scanning) [![Pub Version](https://img.shields.io/pub/v/google_mlkit_barcode_scanning)](https://pub.dev/packages/google_mlkit_barcode_scanning) | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master/packages/google_mlkit_barcode_scanning) | ✅ | ✅ | | [Face Detection](https://developers.google.com/ml-kit/vision/face-detection) | [google\_mlkit\_face\_detection](https://pub.dev/packages/google_mlkit_face_detection) [![Pub Version](https://img.shields.io/pub/v/google_mlkit_face_detection)](https://pub.dev/packages/google_mlkit_face_detection) | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master/packages/google_mlkit_face_detection) | ✅ | ✅ | -| [Face Mesh Detection](https://developers.google.com/ml-kit/vision/face-mesh-detection) | [google\_mlkit\_face\_mesh\_detection](https://pub.dev/packages/google_mlkit_face_mesh_detection) [![Pub Version](https://img.shields.io/pub/v/google_mlkit_face_mesh_detection)](https://pub.dev/packages/google_mlkit_face_mesh_detection) | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master/packages/google_mlkit_face_mesh_detection) | ❌ | ❌ | +| [Face Mesh Detection](https://developers.google.com/ml-kit/vision/face-mesh-detection) | [google\_mlkit\_face\_mesh\_detection](https://pub.dev/packages/google_mlkit_face_mesh_detection) [![Pub Version](https://img.shields.io/pub/v/google_mlkit_face_mesh_detection)](https://pub.dev/packages/google_mlkit_face_mesh_detection) | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master/packages/google_mlkit_face_mesh_detection) | ✅ | ❌ | | [Image Labeling](https://developers.google.com/ml-kit/vision/image-labeling) | [google\_mlkit\_image\_labeling](https://pub.dev/packages/google_mlkit_image_labeling) [![Pub Version](https://img.shields.io/pub/v/google_mlkit_image_labeling)](https://pub.dev/packages/google_mlkit_image_labeling) | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master/packages/google_mlkit_image_labeling) | ✅ | ✅ | | [Object Detection and Tracking](https://developers.google.com/ml-kit/vision/object-detection) | [google\_mlkit\_object\_detection](https://pub.dev/packages/google_mlkit_object_detection) [![Pub Version](https://img.shields.io/pub/v/google_mlkit_object_detection)](https://pub.dev/packages/google_mlkit_object_detection) | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master/packages/google_mlkit_object_detection) | ✅ | ✅ | | [Text Recognition V2](https://developers.google.com/ml-kit/vision/text-recognition/v2) | [google\_mlkit\_text\_recognition](https://pub.dev/packages/google_mlkit_text_recognition) [![Pub Version](https://img.shields.io/pub/v/google_mlkit_text_recognition)](https://pub.dev/packages/google_mlkit_text_recognition) | [![GitHub](https://img.shields.io/badge/github-%23121011.svg?style=for-the-badge&logo=github&logoColor=white)](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master/packages/google_mlkit_text_recognition) | ✅ | ✅ | diff --git a/packages/example/ios/Podfile.lock b/packages/example/ios/Podfile.lock index 1a56cad4..529a4432 100644 --- a/packages/example/ios/Podfile.lock +++ b/packages/example/ios/Podfile.lock @@ -21,6 +21,9 @@ PODS: - Flutter - google_mlkit_commons - GoogleMLKit/FaceDetection (~> 4.0.0) + - google_mlkit_face_mesh_detection (0.0.1): + - Flutter + - google_mlkit_commons - google_mlkit_image_labeling (0.7.0): - Flutter - google_mlkit_commons @@ -280,6 +283,7 @@ DEPENDENCIES: - google_mlkit_digital_ink_recognition (from `.symlinks/plugins/google_mlkit_digital_ink_recognition/ios`) - google_mlkit_entity_extraction (from `.symlinks/plugins/google_mlkit_entity_extraction/ios`) - google_mlkit_face_detection (from `.symlinks/plugins/google_mlkit_face_detection/ios`) + - google_mlkit_face_mesh_detection (from `.symlinks/plugins/google_mlkit_face_mesh_detection/ios`) - google_mlkit_image_labeling (from `.symlinks/plugins/google_mlkit_image_labeling/ios`) - google_mlkit_language_id (from `.symlinks/plugins/google_mlkit_language_id/ios`) - google_mlkit_object_detection (from `.symlinks/plugins/google_mlkit_object_detection/ios`) @@ -353,6 +357,8 @@ EXTERNAL SOURCES: :path: ".symlinks/plugins/google_mlkit_entity_extraction/ios" google_mlkit_face_detection: :path: ".symlinks/plugins/google_mlkit_face_detection/ios" + google_mlkit_face_mesh_detection: + :path: ".symlinks/plugins/google_mlkit_face_mesh_detection/ios" google_mlkit_image_labeling: :path: ".symlinks/plugins/google_mlkit_image_labeling/ios" google_mlkit_language_id: @@ -382,6 +388,7 @@ SPEC CHECKSUMS: google_mlkit_digital_ink_recognition: 07fb2aae2d704592be8f6c2ff63f3adcfd3d768c google_mlkit_entity_extraction: 480e6029eaacf7fac2305276dc111978bd351bb5 google_mlkit_face_detection: 6930c8cecb10f49921ff7010c165eb1d803e87fd + google_mlkit_face_mesh_detection: 2eeca458203c830b42cfb46de71af317079e65d9 google_mlkit_image_labeling: b338655e8ed02823d06503362c754d6b02537824 google_mlkit_language_id: ee26aec812e70084affbcf628f6c5c7308b1291d google_mlkit_object_detection: bab13ee252b52eb3bf96db2e88af30c31dd59588 diff --git a/packages/example/lib/main.dart b/packages/example/lib/main.dart index ca7b8ef9..605a6465 100644 --- a/packages/example/lib/main.dart +++ b/packages/example/lib/main.dart @@ -7,6 +7,7 @@ import 'nlp_detector_views/smart_reply_view.dart'; import 'vision_detector_views/barcode_scanner_view.dart'; import 'vision_detector_views/digital_ink_recognizer_view.dart'; import 'vision_detector_views/face_detector_view.dart'; +import 'vision_detector_views/face_mesh_detector_view.dart'; import 'vision_detector_views/label_detector_view.dart'; import 'vision_detector_views/object_detector_view.dart'; import 'vision_detector_views/pose_detector_view.dart'; @@ -50,6 +51,7 @@ class Home extends StatelessWidget { children: [ CustomCard('Barcode Scanning', BarcodeScannerView()), CustomCard('Face Detection', FaceDetectorView()), + CustomCard('Face Mesh Detection', FaceMeshDetectorView()), CustomCard('Image Labeling', ImageLabelView()), CustomCard('Object Detection', ObjectDetectorView()), CustomCard('Text Recognition', TextRecognizerView()), diff --git a/packages/example/lib/vision_detector_views/face_mesh_detector_view.dart b/packages/example/lib/vision_detector_views/face_mesh_detector_view.dart new file mode 100644 index 00000000..428ce7b0 --- /dev/null +++ b/packages/example/lib/vision_detector_views/face_mesh_detector_view.dart @@ -0,0 +1,84 @@ +import 'dart:io'; + +import 'package:camera/camera.dart'; +import 'package:flutter/material.dart'; +import 'package:google_mlkit_face_mesh_detection/google_mlkit_face_mesh_detection.dart'; + +import 'detector_view.dart'; +import 'painters/face_mesh_detector_painter.dart'; + +class FaceMeshDetectorView extends StatefulWidget { + @override + State createState() => _FaceMeshDetectorViewState(); +} + +class _FaceMeshDetectorViewState extends State { + final FaceMeshDetector _meshDetector = + FaceMeshDetector(option: FaceMeshDetectorOptions.faceMesh); + bool _canProcess = true; + bool _isBusy = false; + CustomPaint? _customPaint; + String? _text; + var _cameraLensDirection = CameraLensDirection.front; + + @override + void dispose() { + _canProcess = false; + _meshDetector.close(); + super.dispose(); + } + + @override + Widget build(BuildContext context) { + if (Platform.isIOS) { + return Scaffold( + appBar: AppBar(title: Text('Under construction')), + body: Center( + child: Text( + 'Not implemented yet for iOS :(\nTry Android', + textAlign: TextAlign.center, + )), + ); + } + return DetectorView( + title: 'Face Mesh Detector', + customPaint: _customPaint, + text: _text, + onImage: _processImage, + initialCameraLensDirection: _cameraLensDirection, + onCameraLensDirectionChanged: (value) => _cameraLensDirection = value, + ); + } + + Future _processImage(InputImage inputImage) async { + if (!_canProcess) return; + if (_isBusy) return; + _isBusy = true; + setState(() { + _text = ''; + }); + final meshes = await _meshDetector.processImage(inputImage); + if (inputImage.metadata?.size != null && + inputImage.metadata?.rotation != null) { + final painter = FaceMeshDetectorPainter( + meshes, + inputImage.metadata!.size, + inputImage.metadata!.rotation, + _cameraLensDirection, + ); + _customPaint = CustomPaint(painter: painter); + } else { + String text = 'Face meshes found: ${meshes.length}\n\n'; + for (final mesh in meshes) { + text += 'face: ${mesh.boundingBox}\n\n'; + } + _text = text; + // TODO: set _customPaint to draw boundingRect on top of image + _customPaint = null; + } + _isBusy = false; + if (mounted) { + setState(() {}); + } + } +} diff --git a/packages/example/lib/vision_detector_views/painters/face_mesh_detector_painter.dart b/packages/example/lib/vision_detector_views/painters/face_mesh_detector_painter.dart new file mode 100644 index 00000000..92a37aa6 --- /dev/null +++ b/packages/example/lib/vision_detector_views/painters/face_mesh_detector_painter.dart @@ -0,0 +1,103 @@ +import 'dart:ui'; + +import 'package:camera/camera.dart'; +import 'package:flutter/material.dart'; +import 'package:google_mlkit_face_mesh_detection/google_mlkit_face_mesh_detection.dart'; + +import 'coordinates_translator.dart'; + +class FaceMeshDetectorPainter extends CustomPainter { + FaceMeshDetectorPainter( + this.meshes, + this.imageSize, + this.rotation, + this.cameraLensDirection, + ); + + final List meshes; + final Size imageSize; + final InputImageRotation rotation; + final CameraLensDirection cameraLensDirection; + + @override + void paint(Canvas canvas, Size size) { + final Paint paint1 = Paint() + ..style = PaintingStyle.stroke + ..strokeWidth = 1.0 + ..color = Colors.red; + final Paint paint2 = Paint() + ..style = PaintingStyle.fill + ..strokeWidth = 1.0 + ..color = Colors.white; + + for (final FaceMesh mesh in meshes) { + final left = translateX( + mesh.boundingBox.left, + size, + imageSize, + rotation, + cameraLensDirection, + ); + final top = translateY( + mesh.boundingBox.top, + size, + imageSize, + rotation, + cameraLensDirection, + ); + final right = translateX( + mesh.boundingBox.right, + size, + imageSize, + rotation, + cameraLensDirection, + ); + final bottom = translateY( + mesh.boundingBox.bottom, + size, + imageSize, + rotation, + cameraLensDirection, + ); + + canvas.drawRect( + Rect.fromLTRB(left, top, right, bottom), + paint1, + ); + + void paintTriangle(FaceMeshTriangle triangle) { + final List cornerPoints = []; + for (final point in triangle.points) { + final double x = translateX( + point.x.toDouble(), + size, + imageSize, + rotation, + cameraLensDirection, + ); + final double y = translateY( + point.y.toDouble(), + size, + imageSize, + rotation, + cameraLensDirection, + ); + + cornerPoints.add(Offset(x, y)); + } + // Add the first point to close the polygon + cornerPoints.add(cornerPoints.first); + canvas.drawPoints(PointMode.polygon, cornerPoints, paint2); + } + + for (final triangle in mesh.triangles) { + paintTriangle(triangle); + } + } + } + + @override + bool shouldRepaint(FaceMeshDetectorPainter oldDelegate) { + return oldDelegate.imageSize != imageSize || oldDelegate.meshes != meshes; + } +} diff --git a/packages/example/pubspec.lock b/packages/example/pubspec.lock index 8a15ec9b..a5b28588 100644 --- a/packages/example/pubspec.lock +++ b/packages/example/pubspec.lock @@ -28,7 +28,7 @@ packages: name: camera_android url: "https://pub.dartlang.org" source: hosted - version: "0.10.8+2" + version: "0.10.8+3" camera_avfoundation: dependency: transitive description: @@ -99,6 +99,34 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "6.1.4" + file_selector_linux: + dependency: transitive + description: + name: file_selector_linux + url: "https://pub.dartlang.org" + source: hosted + version: "0.9.2" + file_selector_macos: + dependency: transitive + description: + name: file_selector_macos + url: "https://pub.dartlang.org" + source: hosted + version: "0.9.3" + file_selector_platform_interface: + dependency: transitive + description: + name: file_selector_platform_interface + url: "https://pub.dartlang.org" + source: hosted + version: "2.6.0" + file_selector_windows: + dependency: transitive + description: + name: file_selector_windows + url: "https://pub.dartlang.org" + source: hosted + version: "0.9.3" flutter: dependency: "direct main" description: flutter @@ -110,7 +138,7 @@ packages: name: flutter_lints url: "https://pub.dartlang.org" source: hosted - version: "2.0.1" + version: "2.0.2" flutter_plugin_android_lifecycle: dependency: transitive description: @@ -163,6 +191,13 @@ packages: relative: true source: path version: "0.7.0" + google_mlkit_face_mesh_detection: + dependency: "direct main" + description: + path: "../google_mlkit_face_mesh_detection" + relative: true + source: path + version: "0.0.1" google_mlkit_image_labeling: dependency: "direct main" description: @@ -239,35 +274,56 @@ packages: name: image_picker url: "https://pub.dartlang.org" source: hosted - version: "0.8.7+5" + version: "0.8.9" image_picker_android: dependency: transitive description: name: image_picker_android url: "https://pub.dartlang.org" source: hosted - version: "0.8.6+17" + version: "0.8.7+3" image_picker_for_web: dependency: transitive description: name: image_picker_for_web url: "https://pub.dartlang.org" source: hosted - version: "2.1.12" + version: "2.2.0" image_picker_ios: dependency: transitive description: name: image_picker_ios url: "https://pub.dartlang.org" source: hosted - version: "0.8.7+4" + version: "0.8.8" + image_picker_linux: + dependency: transitive + description: + name: image_picker_linux + url: "https://pub.dartlang.org" + source: hosted + version: "0.2.1" + image_picker_macos: + dependency: transitive + description: + name: image_picker_macos + url: "https://pub.dartlang.org" + source: hosted + version: "0.2.1" image_picker_platform_interface: dependency: transitive description: name: image_picker_platform_interface url: "https://pub.dartlang.org" source: hosted - version: "2.6.3" + version: "2.8.0" + image_picker_windows: + dependency: transitive + description: + name: image_picker_windows + url: "https://pub.dartlang.org" + source: hosted + version: "0.2.1" js: dependency: transitive description: @@ -303,6 +359,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "1.8.0" + mime: + dependency: transitive + description: + name: mime + url: "https://pub.dartlang.org" + source: hosted + version: "1.0.4" path: dependency: "direct main" description: @@ -351,7 +414,7 @@ packages: name: path_provider_windows url: "https://pub.dartlang.org" source: hosted - version: "2.1.6" + version: "2.1.7" platform: dependency: transitive description: diff --git a/packages/example/pubspec.yaml b/packages/example/pubspec.yaml index fa3e1f0e..3c33993a 100644 --- a/packages/example/pubspec.yaml +++ b/packages/example/pubspec.yaml @@ -13,7 +13,7 @@ environment: dependencies: flutter: sdk: flutter - image_picker: ^0.8.7+5 + image_picker: ^0.8.9 camera: ^0.10.5+2 path: ^1.8.2 path_provider: ^2.0.15 @@ -26,6 +26,8 @@ dependencies: path: ../google_mlkit_digital_ink_recognition google_mlkit_face_detection: path: ../google_mlkit_face_detection + google_mlkit_face_mesh_detection: + path: ../google_mlkit_face_mesh_detection google_mlkit_image_labeling: path: ../google_mlkit_image_labeling google_mlkit_object_detection: @@ -47,7 +49,7 @@ dependencies: path: ../google_mlkit_smart_reply dev_dependencies: - flutter_lints: ^2.0.1 + flutter_lints: ^2.0.2 flutter_test: sdk: flutter diff --git a/packages/google_mlkit_face_mesh_detection/.gitignore b/packages/google_mlkit_face_mesh_detection/.gitignore new file mode 100644 index 00000000..9be145fd --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/.gitignore @@ -0,0 +1,29 @@ +# Miscellaneous +*.class +*.log +*.pyc +*.swp +.DS_Store +.atom/ +.buildlog/ +.history +.svn/ + +# IntelliJ related +*.iml +*.ipr +*.iws +.idea/ + +# The .vscode folder contains launch configuration and tasks you configure in +# VS Code which you may wish to be included in version control, so this line +# is commented out by default. +#.vscode/ + +# Flutter/Dart/Pub related +# Libraries should not include pubspec.lock, per https://dart.dev/guides/libraries/private-files#pubspeclock. +/pubspec.lock +**/doc/api/ +.dart_tool/ +.packages +build/ diff --git a/packages/google_mlkit_face_mesh_detection/CHANGELOG.md b/packages/google_mlkit_face_mesh_detection/CHANGELOG.md new file mode 100644 index 00000000..d0bd041d --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/CHANGELOG.md @@ -0,0 +1,3 @@ +## 0.0.1 + +* Initial release. diff --git a/packages/google_mlkit_face_mesh_detection/LICENSE b/packages/google_mlkit_face_mesh_detection/LICENSE new file mode 100644 index 00000000..07283e46 --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Francisco Bernal and Bharat Biradar. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/google_mlkit_face_mesh_detection/README.md b/packages/google_mlkit_face_mesh_detection/README.md new file mode 100644 index 00000000..5ca0bd20 --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/README.md @@ -0,0 +1,88 @@ +# Google's ML Kit Face Mesh Detection for Flutter + +[![Pub Version](https://img.shields.io/pub/v/google_mlkit_face_mesh_detection)](https://pub.dev/packages/google_mlkit_face_mesh_detection) +[![analysis](https://github.com/flutter-ml/google_ml_kit_flutter/actions/workflows/flutter.yml/badge.svg)](https://github.com/flutter-ml/google_ml_kit_flutter/actions) +[![Star on Github](https://img.shields.io/github/stars/flutter-ml/google_ml_kit_flutter.svg?style=flat&logo=github&colorB=deeppink&label=stars)](https://github.com/flutter-ml/google_ml_kit_flutter) +[![License: MIT](https://img.shields.io/badge/license-MIT-purple.svg)](https://opensource.org/licenses/MIT) + +A Flutter plugin to use [Google's ML Kit Face Mesh Detection](https://developers.google.com/ml-kit/vision/face-mesh-detection) for face mesh detection, you can generate in real-time a high accuracy [face mesh](https://developers.google.com/ml-kit/vision/face-mesh-detection/concepts) of 468 3D points for selfie-like images. + +Faces should be within ~2 meters (~7 feet) of the camera, so that the faces are sufficiently large for optimal face mesh recognition. In general, the larger the face, the better the face mesh recognition. + +If you want to detect faces further than ~2 meters (~7 feet) away from the camera, please see [google_mlkit_face_detection](https://pub.dev/packages/google_mlkit_face_detection). + +Note that the face should be facing the camera with at least half of the face visible. Any large object between the face and the camera may result in lower accuracy. + +**NOTE** Since [Google's Face Mesh Detection](https://developers.google.com/ml-kit/vision/face-mesh-detection) API is still in Beta and only supports Android. Stay tune for updates in their website. + +**PLEASE READ THIS** before continuing or posting a [new issue](https://github.com/flutter-ml/google_ml_kit_flutter/issues): + +- [Google's ML Kit](https://developers.google.com/ml-kit) was build only for mobile platforms: iOS and Android apps. + +- This plugin is not sponsor or maintained by Google. The [authors](https://github.com/flutter-ml/google_ml_kit_flutter/blob/master/AUTHORS) are developers excited about machine learning that wanted to expose Google's native APIs to Flutter. + +- Google's ML Kit APIs are ony developed natively for iOS and Android. This plugin uses Flutter Platform Channels as explained [here](https://docs.flutter.dev/development/platform-integration/platform-channels). + + Messages are passed between the client (the app/plugin) and host (platform) using platform channels as illustrated in this diagram: + +

+ +

+ + Messages and responses are passed asynchronously, to ensure the user interface remains responsive. To read more about platform channels go [here](https://docs.flutter.dev/development/platform-integration/platform-channels). + + Because this plugin uses platform channels, no Machine Learning processing is done in Flutter/Dart, all the calls are passed to the native platform using `MethodChannel` in Android and `FlutterMethodChannel` in iOS, and executed using the Google's native APIs. Think of this plugin as a bridge between your app and Google's native ML Kit APIs. This plugin only passes the call to the native API and the processing is done by Google's API. It is important that you understand this concept when it comes to debugging errors for your ML model and/or app. + +- Since the plugin uses platform channels, you may encounter issues with the native API. Before submitting a new issue, identify the source of the issue. You can run both iOS and/or Android native [example apps by Google](https://github.com/googlesamples/mlkit) and make sure that the issue is not reproducible with their native examples. If you can reproduce the issue in their apps then report the issue to Google. The [authors](https://github.com/flutter-ml/google_ml_kit_flutter/blob/master/AUTHORS) do not have access to the source code of their native APIs, so you need to report the issue to them. If you find that their example apps are okay and still you have an issue using this plugin, then look at our [closed and open issues](https://github.com/flutter-ml/google_ml_kit_flutter/issues). If you cannot find anything that can help you then report the issue and provide enough details. Be patient, someone from the community will eventually help you. + +## Getting Started + +Before you get started read about the requirements and known issues of this plugin [here](https://github.com/flutter-ml/google_ml_kit_flutter#requirements). + +## Usage + +### Face Mesh Detection + +#### Create an instance of `InputImage` + +Create an instance of `InputImage` as explained [here](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master/packages/google_mlkit_commons#creating-an-inputimage). + +```dart +final InputImage inputImage; +``` + +#### Create an instance of `FaceMeshDetector` + +```dart +final meshDetector = FaceMeshDetector(option: FaceMeshDetectorOptions.faceMesh); +``` + +#### Process image + +```dart +final List meshes = await meshDetector.processImage(inputImage); + +for (FaceMesh mesh in meshes) { + final boundingBox = mesh.boundingBox; + final points = mesh.points; + final triangles = mesh.triangles; + final contour = mesh.contours[FaceMeshContourType.faceOval]; +} +``` + +#### Release resources with `close()` + +```dart +meshDetector.close(); +``` + +## Example app + +Find the example app [here](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master/packages/example). + +## Contributing + +Contributions are welcome. +In case of any problems look at [existing issues](https://github.com/flutter-ml/google_ml_kit_flutter/issues), if you cannot find anything related to your problem then open an issue. +Create an issue before opening a [pull request](https://github.com/flutter-ml/google_ml_kit_flutter/pulls) for non trivial fixes. +In case of trivial fixes open a [pull request](https://github.com/flutter-ml/google_ml_kit_flutter/pulls) directly. diff --git a/packages/google_mlkit_face_mesh_detection/analysis_options.yaml b/packages/google_mlkit_face_mesh_detection/analysis_options.yaml new file mode 100644 index 00000000..e2903a86 --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/analysis_options.yaml @@ -0,0 +1 @@ +include: ../google_ml_kit/analysis_options.yaml diff --git a/packages/google_mlkit_face_mesh_detection/android/.gitignore b/packages/google_mlkit_face_mesh_detection/android/.gitignore new file mode 100644 index 00000000..c6cbe562 --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/android/.gitignore @@ -0,0 +1,8 @@ +*.iml +.gradle +/local.properties +/.idea/workspace.xml +/.idea/libraries +.DS_Store +/build +/captures diff --git a/packages/google_mlkit_face_mesh_detection/android/build.gradle b/packages/google_mlkit_face_mesh_detection/android/build.gradle new file mode 100644 index 00000000..d45384a6 --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/android/build.gradle @@ -0,0 +1,39 @@ +group 'com.google_mlkit_face_mesh_detection' +version '1.0' + +buildscript { + repositories { + google() + mavenCentral() + } + + dependencies { + classpath 'com.android.tools.build:gradle:7.2.1' + } +} + +rootProject.allprojects { + repositories { + google() + mavenCentral() + } +} + +apply plugin: 'com.android.library' + +android { + compileSdkVersion 31 + + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } + + defaultConfig { + minSdkVersion 21 + } +} + +dependencies { + implementation 'com.google.mlkit:face-mesh-detection:16.0.0-beta1' +} diff --git a/packages/google_mlkit_face_mesh_detection/android/settings.gradle b/packages/google_mlkit_face_mesh_detection/android/settings.gradle new file mode 100644 index 00000000..fcb8efb6 --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/android/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'google_mlkit_face_mesh_detection' diff --git a/packages/google_mlkit_face_mesh_detection/android/src/main/AndroidManifest.xml b/packages/google_mlkit_face_mesh_detection/android/src/main/AndroidManifest.xml new file mode 100644 index 00000000..44272380 --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/android/src/main/AndroidManifest.xml @@ -0,0 +1,3 @@ + + diff --git a/packages/google_mlkit_face_mesh_detection/android/src/main/java/com/google_mlkit_face_mesh_detection/FaceMeshDetector.java b/packages/google_mlkit_face_mesh_detection/android/src/main/java/com/google_mlkit_face_mesh_detection/FaceMeshDetector.java new file mode 100644 index 00000000..0d39560f --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/android/src/main/java/com/google_mlkit_face_mesh_detection/FaceMeshDetector.java @@ -0,0 +1,159 @@ +package com.google_mlkit_face_mesh_detection; + +import android.content.Context; +import android.graphics.Rect; + +import androidx.annotation.NonNull; + +import com.google.mlkit.vision.common.InputImage; +import com.google.mlkit.vision.common.Triangle; +import com.google.mlkit.vision.facemesh.FaceMesh; +import com.google.mlkit.vision.facemesh.FaceMeshDetection; +import com.google.mlkit.vision.facemesh.FaceMeshDetectorOptions; +import com.google.mlkit.vision.facemesh.FaceMeshPoint; +import com.google_mlkit_commons.InputImageConverter; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import io.flutter.plugin.common.MethodCall; +import io.flutter.plugin.common.MethodChannel; + +class FaceMeshDetector implements MethodChannel.MethodCallHandler { + private static final String START = "vision#startFaceMeshDetector"; + private static final String CLOSE = "vision#closeFaceMeshDetector"; + + private final Context context; + private final Map instances = new HashMap<>(); + + public FaceMeshDetector(Context context) { + this.context = context; + } + + @Override + public void onMethodCall(@NonNull MethodCall call, @NonNull MethodChannel.Result result) { + String method = call.method; + switch (method) { + case START: + handleDetection(call, result); + break; + case CLOSE: + closeDetector(call); + result.success(null); + break; + default: + result.notImplemented(); + break; + } + } + + private void handleDetection(MethodCall call, final MethodChannel.Result result) { + Map imageData = (Map) call.argument("imageData"); + InputImage inputImage = InputImageConverter.getInputImageFromData(imageData, context, result); + if (inputImage == null) return; + + String id = call.argument("id"); + com.google.mlkit.vision.facemesh.FaceMeshDetector detector = instances.get(id); + if (detector == null) { + int option = call.argument("option"); + switch (option) { + case FaceMeshDetectorOptions.BOUNDING_BOX_ONLY: + detector = FaceMeshDetection.getClient( + new FaceMeshDetectorOptions.Builder() + .setUseCase(FaceMeshDetectorOptions.BOUNDING_BOX_ONLY) + .build() + ); + break; + + case FaceMeshDetectorOptions.FACE_MESH: + detector = FaceMeshDetection.getClient(); + + break; + + default: + result.error("FaceMeshDetectorError", "Invalid options", null); + return; + } + + instances.put(id, detector); + } + + detector.process(inputImage) + .addOnSuccessListener( + visionMeshes -> { + List> faceMeshes = new ArrayList<>(visionMeshes.size()); + for (FaceMesh mesh : visionMeshes) { + Map meshData = new HashMap<>(); + + Map frame = new HashMap<>(); + Rect rect = mesh.getBoundingBox(); + frame.put("left", rect.left); + frame.put("top", rect.top); + frame.put("right", rect.right); + frame.put("bottom", rect.bottom); + meshData.put("rect", frame); + + meshData.put("points", pointsToList(mesh.getAllPoints())); + + List>> triangles = new ArrayList<>(); + for (Triangle triangle : mesh.getAllTriangles()) { + triangles.add(pointsToList(triangle.getAllPoints())); + } + meshData.put("triangles", triangles); + + int[] types = { + FaceMesh.FACE_OVAL, + FaceMesh.LEFT_EYEBROW_TOP, + FaceMesh.LEFT_EYEBROW_BOTTOM, + FaceMesh.RIGHT_EYEBROW_TOP, + FaceMesh.RIGHT_EYEBROW_BOTTOM, + FaceMesh.LEFT_EYE, + FaceMesh.RIGHT_EYE, + FaceMesh.UPPER_LIP_TOP, + FaceMesh.UPPER_LIP_BOTTOM, + FaceMesh.LOWER_LIP_TOP, + FaceMesh.LOWER_LIP_BOTTOM, + FaceMesh.NOSE_BRIDGE + }; + Map>> contours = new HashMap<>(); + for (int type : types) { + contours.put(type - 1, pointsToList(mesh.getPoints(type))); + } + meshData.put("contours", contours); + + faceMeshes.add(meshData); + } + + result.success(faceMeshes); + }) + .addOnFailureListener( + e -> result.error("FaceMeshDetectorError", e.toString(), null)); + } + + private List> pointsToList(List points) { + List> list = new ArrayList<>(); + for (FaceMeshPoint point : points) { + list.add(pointToMap(point)); + } + return list; + } + + private Map pointToMap(FaceMeshPoint point) { + Map pointMap = new HashMap<>(); + pointMap.put("index", point.getIndex()); + pointMap.put("x", point.getPosition().getX()); + pointMap.put("y", point.getPosition().getY()); + pointMap.put("z", point.getPosition().getZ()); + return pointMap; + } + + private void closeDetector(MethodCall call) { + String id = call.argument("id"); + com.google.mlkit.vision.facemesh.FaceMeshDetector detector = instances.get(id); + if (detector == null) return; + detector.close(); + instances.remove(id); + } +} diff --git a/packages/google_mlkit_face_mesh_detection/android/src/main/java/com/google_mlkit_face_mesh_detection/GoogleMlKitFaceMeshDetectionPlugin.java b/packages/google_mlkit_face_mesh_detection/android/src/main/java/com/google_mlkit_face_mesh_detection/GoogleMlKitFaceMeshDetectionPlugin.java new file mode 100644 index 00000000..56bedce1 --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/android/src/main/java/com/google_mlkit_face_mesh_detection/GoogleMlKitFaceMeshDetectionPlugin.java @@ -0,0 +1,22 @@ +package com.google_mlkit_face_mesh_detection; + +import androidx.annotation.NonNull; + +import io.flutter.embedding.engine.plugins.FlutterPlugin; +import io.flutter.plugin.common.MethodChannel; + +public class GoogleMlKitFaceMeshDetectionPlugin implements FlutterPlugin { + private MethodChannel channel; + private static final String channelName = "google_mlkit_face_mesh_detector"; + + @Override + public void onAttachedToEngine(@NonNull FlutterPluginBinding flutterPluginBinding) { + channel = new MethodChannel(flutterPluginBinding.getBinaryMessenger(), channelName); + channel.setMethodCallHandler(new FaceMeshDetector(flutterPluginBinding.getApplicationContext())); + } + + @Override + public void onDetachedFromEngine(@NonNull FlutterPluginBinding binding) { + channel.setMethodCallHandler(null); + } +} diff --git a/packages/google_mlkit_face_mesh_detection/example/README.md b/packages/google_mlkit_face_mesh_detection/example/README.md new file mode 100644 index 00000000..004204a3 --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/example/README.md @@ -0,0 +1,3 @@ +## Example app + +Find the example app [here](https://github.com/flutter-ml/google_ml_kit_flutter/tree/master/packages/example). diff --git a/packages/google_mlkit_face_mesh_detection/ios/.gitignore b/packages/google_mlkit_face_mesh_detection/ios/.gitignore new file mode 100644 index 00000000..0c885071 --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/ios/.gitignore @@ -0,0 +1,38 @@ +.idea/ +.vagrant/ +.sconsign.dblite +.svn/ + +.DS_Store +*.swp +profile + +DerivedData/ +build/ +GeneratedPluginRegistrant.h +GeneratedPluginRegistrant.m + +.generated/ + +*.pbxuser +*.mode1v3 +*.mode2v3 +*.perspectivev3 + +!default.pbxuser +!default.mode1v3 +!default.mode2v3 +!default.perspectivev3 + +xcuserdata + +*.moved-aside + +*.pyc +*sync/ +Icon? +.tags* + +/Flutter/Generated.xcconfig +/Flutter/ephemeral/ +/Flutter/flutter_export_environment.sh \ No newline at end of file diff --git a/packages/google_mlkit_face_mesh_detection/ios/Assets/.gitkeep b/packages/google_mlkit_face_mesh_detection/ios/Assets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/packages/google_mlkit_face_mesh_detection/ios/Classes/GoogleMlKitFaceMeshDetectionPlugin.h b/packages/google_mlkit_face_mesh_detection/ios/Classes/GoogleMlKitFaceMeshDetectionPlugin.h new file mode 100644 index 00000000..7677fc0f --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/ios/Classes/GoogleMlKitFaceMeshDetectionPlugin.h @@ -0,0 +1,4 @@ +#import + +@interface GoogleMlKitFaceMeshDetectionPlugin : NSObject +@end diff --git a/packages/google_mlkit_face_mesh_detection/ios/Classes/GoogleMlKitFaceMeshDetectionPlugin.m b/packages/google_mlkit_face_mesh_detection/ios/Classes/GoogleMlKitFaceMeshDetectionPlugin.m new file mode 100644 index 00000000..df513652 --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/ios/Classes/GoogleMlKitFaceMeshDetectionPlugin.m @@ -0,0 +1,45 @@ +#import "GoogleMlKitFaceMeshDetectionPlugin.h" +#import + +#define channelName @"google_mlkit_face_mesh_detector" +#define startFaceMeshDetector @"vision#startFaceMeshDetector" +#define closeFaceMeshDetector @"vision#closeFaceMeshDetector" + +@implementation GoogleMlKitFaceMeshDetectionPlugin { + NSMutableDictionary *instances; +} + ++ (void)registerWithRegistrar:(NSObject*)registrar { + FlutterMethodChannel* channel = [FlutterMethodChannel + methodChannelWithName:channelName + binaryMessenger:[registrar messenger]]; + GoogleMlKitFaceMeshDetectionPlugin* instance = [[GoogleMlKitFaceMeshDetectionPlugin alloc] init]; + [registrar addMethodCallDelegate:instance channel:channel]; +} + +- (id)init { + self = [super init]; + if (self) + instances = [NSMutableDictionary dictionary]; + return self; +} + +- (void)handleMethodCall:(FlutterMethodCall *)call result:(FlutterResult)result { + if ([call.method isEqualToString:startFaceMeshDetector]) { + [self handleDetection:call result:result]; + } else if ([call.method isEqualToString:closeFaceMeshDetector]) { + NSString *uid = call.arguments[@"id"]; + [instances removeObjectForKey:uid]; + result(NULL); + } else { + result(FlutterMethodNotImplemented); + } +} + +- (void)handleDetection:(FlutterMethodCall *)call result:(FlutterResult)result { + // TODO: waiting for Google to release Face Mesh api for iOS + // https://developers.google.com/ml-kit/vision/face-mesh-detection + result(FlutterMethodNotImplemented); +} + +@end diff --git a/packages/google_mlkit_face_mesh_detection/ios/google_mlkit_face_mesh_detection.podspec b/packages/google_mlkit_face_mesh_detection/ios/google_mlkit_face_mesh_detection.podspec new file mode 100644 index 00000000..9be4d55c --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/ios/google_mlkit_face_mesh_detection.podspec @@ -0,0 +1,27 @@ +require 'yaml' + +pubspec = YAML.load_file(File.join('..', 'pubspec.yaml')) +library_version = pubspec['version'].gsub('+', '-') + +Pod::Spec.new do |s| + s.name = pubspec['name'] + s.version = library_version + s.summary = pubspec['description'] + s.description = pubspec['description'] + s.homepage = pubspec['homepage'] + s.license = { :file => '../LICENSE' } + s.authors = 'Multiple Authors' + s.source = { :path => '.' } + s.source_files = 'Classes/**/*' + s.public_header_files = 'Classes/**/*.h' + s.dependency 'Flutter' +# s.dependency 'GoogleMLKit/FaceMeshDetection', '~> 4.0.0' + s.dependency 'google_mlkit_commons' + s.platform = :ios, '10.0' + s.ios.deployment_target = '10.0' + s.static_framework = true + s.swift_version = '5.0' + + # Flutter.framework does not contain a i386 slice. + s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES', 'EXCLUDED_ARCHS[sdk=iphonesimulator*]' => 'i386' } +end diff --git a/packages/google_mlkit_face_mesh_detection/lib/google_mlkit_face_mesh_detection.dart b/packages/google_mlkit_face_mesh_detection/lib/google_mlkit_face_mesh_detection.dart new file mode 100644 index 00000000..832d055a --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/lib/google_mlkit_face_mesh_detection.dart @@ -0,0 +1,3 @@ +export 'package:google_mlkit_commons/google_mlkit_commons.dart'; + +export 'src/face_mesh_detector.dart'; diff --git a/packages/google_mlkit_face_mesh_detection/lib/src/face_mesh_detector.dart b/packages/google_mlkit_face_mesh_detection/lib/src/face_mesh_detector.dart new file mode 100644 index 00000000..d47aded9 --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/lib/src/face_mesh_detector.dart @@ -0,0 +1,168 @@ +import 'package:flutter/material.dart'; + +import 'package:flutter/services.dart' as services; +import 'package:google_mlkit_commons/google_mlkit_commons.dart'; + +/// A face mesh detector that detects a face mesh in a given [InputImage]. +class FaceMeshDetector { + static const services.MethodChannel _channel = + services.MethodChannel('google_mlkit_face_mesh_detector'); + + /// Instance id. + final id = DateTime.now().microsecondsSinceEpoch.toString(); + + /// Options for [FaceMeshDetector]. + final FaceMeshDetectorOptions option; + + /// Constructor to create an instance of [FaceMeshDetector]. + FaceMeshDetector({required this.option}); + + /// Processes the given image for face mesh detection. + Future> processImage(InputImage inputImage) async { + final result = await _channel.invokeListMethod( + 'vision#startFaceMeshDetector', { + 'id': id, + 'option': option.index, + 'imageData': inputImage.toJson(), + }); + + final List meshes = []; + for (final dynamic json in result!) { + meshes.add(FaceMesh.fromJson(json)); + } + + return meshes; + } + + /// Closes the detector and releases its resources. + Future close() => + _channel.invokeMethod('vision#closeFaceMeshDetector', {'id': id}); +} + +/// Represent face mesh detected by [FaceMeshDetector]. +class FaceMesh { + /// Returns the NonNull axis-aligned bounding rectangle of the detected face mesh. + final Rect boundingBox; + + /// Returns a list of [FaceMeshPoint] representing the whole detected face. + final List points; + + /// Returns a list of [FaceMeshTriangle] representing logical triangle surfaces of detected face. + final List triangles; + + /// Returns a map with lists of FaceMeshPoint representing a specific contour. + final Map?> contours; + + /// Creates a face mesh. + FaceMesh( + {required this.boundingBox, + required this.points, + required this.triangles, + required this.contours}); + + /// Returns an instance of [FaceMesh] from a given [json]. + factory FaceMesh.fromJson(Map json) => FaceMesh( + boundingBox: RectJson.fromJson(json['rect']), + points: json['points'] + .map((element) { + return FaceMeshPoint.fromJson(element); + }) + .cast() + .toList(), + triangles: json['triangles'] + .map((element) { + return FaceMeshTriangle.fromJson(element); + }) + .cast() + .toList(), + contours: Map>.fromIterables( + FaceMeshContourType.values, + FaceMeshContourType.values.map((FaceMeshContourType type) { + final List? arr = (json['contours'] ?? {})[type.index]; + return (arr == null) + ? [] + : arr + .map((element) { + return FaceMeshPoint.fromJson(element); + }) + .cast() + .toList(); + })), + ); +} + +/// Represents a 3D point in face mesh, by index and PointF3D. +class FaceMeshPoint { + /// Gets the index of the face mesh point, ranging from 0 to 467. + final int index; + + /// Returns the X value of the point. + final double x; + + /// Returns the Y value of the point. + final double y; + + /// Returns the Z value of the point. + final double z; + + /// Creates a face mesh point. + FaceMeshPoint({ + required this.index, + required this.x, + required this.y, + required this.z, + }); + + /// Returns an instance of [FaceMeshPoint] from a given [json]. + factory FaceMeshPoint.fromJson(Map json) => FaceMeshPoint( + index: json['index'], + x: json['x'], + y: json['y'], + z: json['z'], + ); +} + +/// Represents a triangle with 3 generic points. +class FaceMeshTriangle { + /// Gets all points inside the [FaceMeshTriangle]. + final List points; + + /// Creates a triangle with 3 generic points. + FaceMeshTriangle({required this.points}); + + /// Returns an instance of [FaceMeshTriangle] from a given [json]. + factory FaceMeshTriangle.fromJson(List json) => FaceMeshTriangle( + points: json + .map((element) { + return FaceMeshPoint.fromJson(element); + }) + .cast() + .toList()); +} + +/// Options for [FaceMeshDetector]. +enum FaceMeshDetectorOptions { + /// Only provides a bounding box for a detected face mesh. + /// This is the fastest face detector, but has with range limitation(faces must be within ~2 meters or ~7 feet of the camera). + boundingBoxOnly, + + /// Provides a bounding box and additional face mesh info (468 3D points and triangle info). + /// When compared to the [boundingBoxOnly] use case, latency increases by ~15%. + faceMesh, +} + +/// Type of face mesh contour. +enum FaceMeshContourType { + faceOval, + leftEyebrowTop, + leftEyebrowBottom, + rightEyebrowTop, + rightEyebrowBottom, + leftEye, + rightEye, + upperLipTop, + upperLipBottom, + lowerLipTop, + lowerLipBottom, + noseBridge, +} diff --git a/packages/google_mlkit_face_mesh_detection/pubspec.yaml b/packages/google_mlkit_face_mesh_detection/pubspec.yaml new file mode 100644 index 00000000..fc11a220 --- /dev/null +++ b/packages/google_mlkit_face_mesh_detection/pubspec.yaml @@ -0,0 +1,28 @@ +name: google_mlkit_face_mesh_detection +description: A Flutter plugin to use Google's ML Kit Face Mesh Detection. +version: 0.0.1 +homepage: https://github.com/flutter-ml/google_ml_kit_flutter + +environment: + sdk: ">=2.16.0 <3.0.0" + flutter: ">=2.5.0" + +dependencies: + flutter: + sdk: flutter + google_mlkit_commons: ^0.4.0 +# path: ../google_mlkit_commons + +dev_dependencies: + flutter_test: + sdk: flutter + flutter_lints: ^2.0.2 + +flutter: + plugin: + platforms: + android: + package: com.google_mlkit_face_mesh_detection + pluginClass: GoogleMlKitFaceMeshDetectionPlugin + ios: + pluginClass: GoogleMlKitFaceMeshDetectionPlugin diff --git a/update_libs.sh b/update_libs.sh index d93e119d..12d8e6ed 100755 --- a/update_libs.sh +++ b/update_libs.sh @@ -17,6 +17,9 @@ flutter pub get cd ../google_mlkit_face_detection flutter pub get +cd ../google_mlkit_face_mesh_detection +flutter pub get + cd ../google_mlkit_image_labeling flutter pub get