From 8ddef51ff94b9e2036277a92f500ba220573c525 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Fri, 13 Oct 2023 09:10:34 -0700 Subject: [PATCH 01/13] adds mediapipe_core package --- packages/mediapipe-core/.gitignore | 7 + packages/mediapipe-core/CHANGELOG.md | 3 + packages/mediapipe-core/README.md | 39 ++ packages/mediapipe-core/analysis_options.yaml | 24 ++ packages/mediapipe-core/ffigen.yaml | 13 + .../lib/generated/core_symbols.yaml | 38 ++ .../mediapipe-core/lib/mediapipe_core.dart | 5 + .../mediapipe-core/lib/src/containers.dart | 109 ++++++ .../mediapipe-core/lib/src/ffi_utils.dart | 52 +++ .../lib/src/mediapipe_common_bindings.dart | 332 ++++++++++++++++++ .../mediapipe-core/lib/src/task_options.dart | 120 +++++++ packages/mediapipe-core/pubspec.yaml | 17 + .../test/task_options_test.dart | 116 ++++++ .../tasks/c/components/containers/category.h | 50 +++ .../containers/classification_result.h | 68 ++++ .../processors/classifier_options.h | 59 ++++ .../mediapipe/tasks/c/core/base_options.h | 36 ++ 17 files changed, 1088 insertions(+) create mode 100644 packages/mediapipe-core/.gitignore create mode 100644 packages/mediapipe-core/CHANGELOG.md create mode 100644 packages/mediapipe-core/README.md create mode 100644 packages/mediapipe-core/analysis_options.yaml create mode 100644 packages/mediapipe-core/ffigen.yaml create mode 100644 packages/mediapipe-core/lib/generated/core_symbols.yaml create mode 100644 packages/mediapipe-core/lib/mediapipe_core.dart create mode 100644 packages/mediapipe-core/lib/src/containers.dart create mode 100644 packages/mediapipe-core/lib/src/ffi_utils.dart create mode 100644 packages/mediapipe-core/lib/src/mediapipe_common_bindings.dart create mode 100644 packages/mediapipe-core/lib/src/task_options.dart create mode 100644 packages/mediapipe-core/pubspec.yaml create mode 100644 packages/mediapipe-core/test/task_options_test.dart create mode 100644 packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/category.h create mode 100644 packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h create mode 100644 packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h create mode 100644 packages/mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h diff --git a/packages/mediapipe-core/.gitignore b/packages/mediapipe-core/.gitignore new file mode 100644 index 0000000..3cceda5 --- /dev/null +++ b/packages/mediapipe-core/.gitignore @@ -0,0 +1,7 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ + +# Avoid committing pubspec.lock for library packages; see +# https://dart.dev/guides/libraries/private-files#pubspeclock. +pubspec.lock diff --git a/packages/mediapipe-core/CHANGELOG.md b/packages/mediapipe-core/CHANGELOG.md new file mode 100644 index 0000000..effe43c --- /dev/null +++ b/packages/mediapipe-core/CHANGELOG.md @@ -0,0 +1,3 @@ +## 1.0.0 + +- Initial version. diff --git a/packages/mediapipe-core/README.md b/packages/mediapipe-core/README.md new file mode 100644 index 0000000..8b55e73 --- /dev/null +++ b/packages/mediapipe-core/README.md @@ -0,0 +1,39 @@ + + +TODO: Put a short description of the package here that helps potential users +know whether this package might be useful for them. + +## Features + +TODO: List what your package can do. Maybe include images, gifs, or videos. + +## Getting started + +TODO: List prerequisites and provide or point to information on how to +start using the package. + +## Usage + +TODO: Include short and useful examples for package users. Add longer examples +to `/example` folder. + +```dart +const like = 'sample'; +``` + +## Additional information + +TODO: Tell users more about the package: where to find more information, how to +contribute to the package, how to file issues, what response they can expect +from the package authors, and more. diff --git a/packages/mediapipe-core/analysis_options.yaml b/packages/mediapipe-core/analysis_options.yaml new file mode 100644 index 0000000..0e9249f --- /dev/null +++ b/packages/mediapipe-core/analysis_options.yaml @@ -0,0 +1,24 @@ +# This file configures the static analysis results for your project (errors, +# warnings, and lints). +# +# This enables the 'recommended' set of lints from `package:lints`. +# This set helps identify many issues that may lead to problems when running +# or consuming Dart code, and enforces writing Dart using a single, idiomatic +# style and format. +# +# If you want a smaller set of lints you can change this to specify +# 'package:lints/core.yaml'. These are just the most critical lints +# (the recommended set includes the core lints). +# The core lints are also what is used by pub.dev for scoring packages. + +include: package:lints/recommended.yaml + +# Uncomment the following section to specify additional rules. + +# linter: +# rules: +# - camel_case_types + +analyzer: + exclude: + - "lib/src/mediapipe_common_bindings.dart" diff --git a/packages/mediapipe-core/ffigen.yaml b/packages/mediapipe-core/ffigen.yaml new file mode 100644 index 0000000..10c8a88 --- /dev/null +++ b/packages/mediapipe-core/ffigen.yaml @@ -0,0 +1,13 @@ +name: "MediaPipeCommonBindings" +description: "Bindings for shared MediaPipe structs common across many tasks" +output: + bindings: "lib/src/mediapipe_common_bindings.dart" + symbol-file: + output: "package:mediapipe_core/generated/core_symbols.yaml" + import-path: "package:mediapipe_core/src/mediapipe_common_bindings.dart" +headers: + entry-points: + - "third_party/mediapipe/tasks/c/core/base_options.h" + - "third_party/mediapipe/tasks/c/components/containers/category.h" + - "third_party/mediapipe/tasks/c/components/containers/classification_result.h" + - "third_party/mediapipe/tasks/c/components/processors/classifier_options.h" diff --git a/packages/mediapipe-core/lib/generated/core_symbols.yaml b/packages/mediapipe-core/lib/generated/core_symbols.yaml new file mode 100644 index 0000000..497d7c3 --- /dev/null +++ b/packages/mediapipe-core/lib/generated/core_symbols.yaml @@ -0,0 +1,38 @@ +format_version: 1.0.0 +files: + package:mediapipe_core/src/mediapipe_common_bindings.dart: + used-config: + ffi-native: false + symbols: + c:@S@BaseOptions: + name: BaseOptions + c:@S@Category: + name: Category + c:@S@ClassificationResult: + name: ClassificationResult + c:@S@Classifications: + name: Classifications + c:@S@ClassifierOptions: + name: ClassifierOptions + c:@S@__darwin_pthread_handler_rec: + name: __darwin_pthread_handler_rec + c:@S@_opaque_pthread_attr_t: + name: _opaque_pthread_attr_t + c:@S@_opaque_pthread_cond_t: + name: _opaque_pthread_cond_t + c:@S@_opaque_pthread_condattr_t: + name: _opaque_pthread_condattr_t + c:@S@_opaque_pthread_mutex_t: + name: _opaque_pthread_mutex_t + c:@S@_opaque_pthread_mutexattr_t: + name: _opaque_pthread_mutexattr_t + c:@S@_opaque_pthread_once_t: + name: _opaque_pthread_once_t + c:@S@_opaque_pthread_rwlock_t: + name: _opaque_pthread_rwlock_t + c:@S@_opaque_pthread_rwlockattr_t: + name: _opaque_pthread_rwlockattr_t + c:@S@_opaque_pthread_t: + name: _opaque_pthread_t + c:@UA@__mbstate_t: + name: __mbstate_t diff --git a/packages/mediapipe-core/lib/mediapipe_core.dart b/packages/mediapipe-core/lib/mediapipe_core.dart new file mode 100644 index 0000000..2e62b18 --- /dev/null +++ b/packages/mediapipe-core/lib/mediapipe_core.dart @@ -0,0 +1,5 @@ +library; + +export 'src/containers.dart' show Category, Classifications; +export 'src/ffi_utils.dart'; +export 'src/task_options.dart' show BaseOptions, ClassifierOptions; diff --git a/packages/mediapipe-core/lib/src/containers.dart b/packages/mediapipe-core/lib/src/containers.dart new file mode 100644 index 0000000..f8f66e4 --- /dev/null +++ b/packages/mediapipe-core/lib/src/containers.dart @@ -0,0 +1,109 @@ +import 'dart:ffi'; +import 'package:ffi/ffi.dart'; +import 'package:mediapipe_core/mediapipe_core.dart'; +import 'mediapipe_common_bindings.dart' as bindings; + +class Category { + const Category({ + required this.index, + required this.score, + required this.categoryName, + required this.displayName, + }); + final int index; + final double score; + final String? categoryName; + final String? displayName; + + static List fromStructs( + Pointer structs, + int count, + ) { + final categories = []; + for (int i = 0; i < count; i++) { + categories.add(fromStruct(structs[i])); + } + return categories; + } + + static Category fromStruct(bindings.Category struct) { + return Category( + index: struct.index, + score: struct.score, + categoryName: toDartString(struct.category_name), + displayName: toDartString(struct.display_name), + ); + } + + static void freeStructs(Pointer structs, int count) { + int index = 0; + while (index < count) { + bindings.Category obj = structs[index]; + calloc.free(obj.category_name); + calloc.free(obj.display_name); + index++; + } + calloc.free(structs); + } + + @override + String toString() => 'Category(index=$index, score=$score, ' + 'categoryName=$categoryName, displayName=$displayName)'; +} + +class Classifications { + const Classifications({ + required this.categories, + required this.headIndex, + required this.headName, + }); + final List categories; + final int headIndex; + final String? headName; + + static List fromStructs( + Pointer structs, + int count, + ) { + final classifications = []; + for (int i = 0; i < count; i++) { + classifications.add(fromStruct(structs[i])); + } + return classifications; + } + + static Classifications fromStruct(bindings.Classifications struct) { + return Classifications( + categories: Category.fromStructs( + struct.categories, + struct.categories_count, + ), + headIndex: struct.head_index, + headName: toDartString(struct.head_name), + ); + } + + static void freeStructs( + Pointer structs, + int count, + ) { + int index = 0; + while (index < count) { + bindings.Classifications obj = structs[index]; + Category.freeStructs(obj.categories, obj.categories_count); + calloc.free(obj.head_name); + index++; + } + calloc.free(structs); + } + + Category? get firstCategory => + categories.isNotEmpty ? categories.first : null; + + @override + String toString() { + final categoryStrings = categories.map((cat) => cat.toString()).join(', '); + return 'Classification(categories=[$categoryStrings], ' + 'headIndex=$headIndex, headName=$headName)'; + } +} diff --git a/packages/mediapipe-core/lib/src/ffi_utils.dart b/packages/mediapipe-core/lib/src/ffi_utils.dart new file mode 100644 index 0000000..99327d8 --- /dev/null +++ b/packages/mediapipe-core/lib/src/ffi_utils.dart @@ -0,0 +1,52 @@ +import 'dart:ffi'; +import 'dart:typed_data'; +import 'package:ffi/ffi.dart'; + +Pointer> prepareListOfStrings(List values) { + final ptrArray = calloc>(values.length); + for (var i = 0; i < values.length; i++) { + ptrArray[i] = values[i].toNativeUtf8().cast(); + } + return ptrArray; +} + +Pointer prepareString(String val) => val.toNativeUtf8().cast(); + +String? toDartString(Pointer val) { + if (val == nullptr) return null; + return val.cast().toDartString(); +} + +List toDartStrings(Pointer> val, int length) { + final dartStrings = []; + int counter = 0; + while (counter < length) { + dartStrings.add(toDartString(val[counter])); + counter++; + } + return dartStrings; +} + +Pointer prepareUint8List(Uint8List ints) { + final Pointer ptr = calloc(ints.length); + ptr.asTypedList(ints.length).setAll(0, ints); + return ptr.cast(); +} + +Uint8List toUint8List(Pointer val, {int? length}) { + final codeUnits = val.cast(); + if (length != null) { + RangeError.checkNotNegative(length, 'length'); + } else { + length = _length(codeUnits); + } + return codeUnits.asTypedList(length); +} + +int _length(Pointer codeUnits) { + var length = 0; + while (codeUnits[length] != 0) { + length++; + } + return length; +} diff --git a/packages/mediapipe-core/lib/src/mediapipe_common_bindings.dart b/packages/mediapipe-core/lib/src/mediapipe_common_bindings.dart new file mode 100644 index 0000000..839b08a --- /dev/null +++ b/packages/mediapipe-core/lib/src/mediapipe_common_bindings.dart @@ -0,0 +1,332 @@ +// AUTO GENERATED FILE, DO NOT EDIT. +// +// Generated by `package:ffigen`. +// ignore_for_file: type=lint +import 'dart:ffi' as ffi; + +final class BaseOptions extends ffi.Struct { + external ffi.Pointer model_asset_buffer; + + external ffi.Pointer model_asset_path; +} + +final class Category extends ffi.Struct { + @ffi.Int() + external int index; + + @ffi.Float() + external double score; + + external ffi.Pointer category_name; + + external ffi.Pointer display_name; +} + +final class __mbstate_t extends ffi.Union { + @ffi.Array.multi([128]) + external ffi.Array __mbstate8; + + @ffi.LongLong() + external int _mbstateL; +} + +final class __darwin_pthread_handler_rec extends ffi.Struct { + external ffi + .Pointer)>> + __routine; + + external ffi.Pointer __arg; + + external ffi.Pointer<__darwin_pthread_handler_rec> __next; +} + +final class _opaque_pthread_attr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([56]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_cond_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([40]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_condattr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([8]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_mutex_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([56]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_mutexattr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([8]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_once_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([8]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_rwlock_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([192]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_rwlockattr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([16]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + external ffi.Pointer<__darwin_pthread_handler_rec> __cleanup_stack; + + @ffi.Array.multi([8176]) + external ffi.Array __opaque; +} + +final class Classifications extends ffi.Struct { + external ffi.Pointer categories; + + @ffi.Uint32() + external int categories_count; + + @ffi.Int() + external int head_index; + + external ffi.Pointer head_name; +} + +final class ClassificationResult extends ffi.Struct { + external ffi.Pointer classifications; + + @ffi.Uint32() + external int classifications_count; + + @ffi.Int64() + external int timestamp_ms; + + @ffi.Bool() + external bool has_timestamp_ms; +} + +final class ClassifierOptions extends ffi.Struct { + external ffi.Pointer display_names_locale; + + @ffi.Int() + external int max_results; + + @ffi.Float() + external double score_threshold; + + external ffi.Pointer> category_allowlist; + + @ffi.Uint32() + external int category_allowlist_count; + + external ffi.Pointer> category_denylist; + + @ffi.Uint32() + external int category_denylist_count; +} + +const int true1 = 1; + +const int false1 = 0; + +const int __bool_true_false_are_defined = 1; + +const int __WORDSIZE = 64; + +const int __DARWIN_ONLY_64_BIT_INO_T = 1; + +const int __DARWIN_ONLY_UNIX_CONFORMANCE = 1; + +const int __DARWIN_ONLY_VERS_1050 = 1; + +const int __DARWIN_UNIX03 = 1; + +const int __DARWIN_64_BIT_INO_T = 1; + +const int __DARWIN_VERS_1050 = 1; + +const int __DARWIN_NON_CANCELABLE = 0; + +const String __DARWIN_SUF_EXTSN = '\$DARWIN_EXTSN'; + +const int __DARWIN_C_ANSI = 4096; + +const int __DARWIN_C_FULL = 900000; + +const int __DARWIN_C_LEVEL = 900000; + +const int __STDC_WANT_LIB_EXT1__ = 1; + +const int __DARWIN_NO_LONG_LONG = 0; + +const int _DARWIN_FEATURE_64_BIT_INODE = 1; + +const int _DARWIN_FEATURE_ONLY_64_BIT_INODE = 1; + +const int _DARWIN_FEATURE_ONLY_VERS_1050 = 1; + +const int _DARWIN_FEATURE_ONLY_UNIX_CONFORMANCE = 1; + +const int _DARWIN_FEATURE_UNIX_CONFORMANCE = 3; + +const int __has_ptrcheck = 0; + +const int __DARWIN_NULL = 0; + +const int __PTHREAD_SIZE__ = 8176; + +const int __PTHREAD_ATTR_SIZE__ = 56; + +const int __PTHREAD_MUTEXATTR_SIZE__ = 8; + +const int __PTHREAD_MUTEX_SIZE__ = 56; + +const int __PTHREAD_CONDATTR_SIZE__ = 8; + +const int __PTHREAD_COND_SIZE__ = 40; + +const int __PTHREAD_ONCE_SIZE__ = 8; + +const int __PTHREAD_RWLOCK_SIZE__ = 192; + +const int __PTHREAD_RWLOCKATTR_SIZE__ = 16; + +const int USER_ADDR_NULL = 0; + +const int INT8_MAX = 127; + +const int INT16_MAX = 32767; + +const int INT32_MAX = 2147483647; + +const int INT64_MAX = 9223372036854775807; + +const int INT8_MIN = -128; + +const int INT16_MIN = -32768; + +const int INT32_MIN = -2147483648; + +const int INT64_MIN = -9223372036854775808; + +const int UINT8_MAX = 255; + +const int UINT16_MAX = 65535; + +const int UINT32_MAX = 4294967295; + +const int UINT64_MAX = -1; + +const int INT_LEAST8_MIN = -128; + +const int INT_LEAST16_MIN = -32768; + +const int INT_LEAST32_MIN = -2147483648; + +const int INT_LEAST64_MIN = -9223372036854775808; + +const int INT_LEAST8_MAX = 127; + +const int INT_LEAST16_MAX = 32767; + +const int INT_LEAST32_MAX = 2147483647; + +const int INT_LEAST64_MAX = 9223372036854775807; + +const int UINT_LEAST8_MAX = 255; + +const int UINT_LEAST16_MAX = 65535; + +const int UINT_LEAST32_MAX = 4294967295; + +const int UINT_LEAST64_MAX = -1; + +const int INT_FAST8_MIN = -128; + +const int INT_FAST16_MIN = -32768; + +const int INT_FAST32_MIN = -2147483648; + +const int INT_FAST64_MIN = -9223372036854775808; + +const int INT_FAST8_MAX = 127; + +const int INT_FAST16_MAX = 32767; + +const int INT_FAST32_MAX = 2147483647; + +const int INT_FAST64_MAX = 9223372036854775807; + +const int UINT_FAST8_MAX = 255; + +const int UINT_FAST16_MAX = 65535; + +const int UINT_FAST32_MAX = 4294967295; + +const int UINT_FAST64_MAX = -1; + +const int INTPTR_MAX = 9223372036854775807; + +const int INTPTR_MIN = -9223372036854775808; + +const int UINTPTR_MAX = -1; + +const int INTMAX_MAX = 9223372036854775807; + +const int UINTMAX_MAX = -1; + +const int INTMAX_MIN = -9223372036854775808; + +const int PTRDIFF_MIN = -9223372036854775808; + +const int PTRDIFF_MAX = 9223372036854775807; + +const int SIZE_MAX = -1; + +const int RSIZE_MAX = 9223372036854775807; + +const int WCHAR_MAX = 2147483647; + +const int WCHAR_MIN = -2147483648; + +const int WINT_MIN = -2147483648; + +const int WINT_MAX = 2147483647; + +const int SIG_ATOMIC_MIN = -2147483648; + +const int SIG_ATOMIC_MAX = 2147483647; diff --git a/packages/mediapipe-core/lib/src/task_options.dart b/packages/mediapipe-core/lib/src/task_options.dart new file mode 100644 index 0000000..2e1320c --- /dev/null +++ b/packages/mediapipe-core/lib/src/task_options.dart @@ -0,0 +1,120 @@ +import 'dart:ffi'; +import 'dart:typed_data'; +import 'package:equatable/equatable.dart'; +import 'package:ffi/ffi.dart'; +import 'mediapipe_common_bindings.dart' as bindings; +import 'ffi_utils.dart'; + +class BaseOptions extends Equatable { + const BaseOptions({this.modelAssetBuffer, this.modelAssetPath}) + : assert( + !(modelAssetBuffer == null && modelAssetPath == null), + 'You must supply either `modelAssetBuffer` or `modelAssetPath`', + ), + assert( + !(modelAssetBuffer != null && modelAssetPath != null), + 'You must only supply one of `modelAssetBuffer` and `modelAssetPath`', + ); + + /// The model asset file contents as bytes; + final Uint8List? modelAssetBuffer; + + /// Path to the model asset file. + final String? modelAssetPath; + + Pointer toStruct() { + final struct = calloc(); + + if (modelAssetPath != null) { + struct.ref.model_asset_path = prepareString(modelAssetPath!); + } + if (modelAssetBuffer != null) { + struct.ref.model_asset_buffer = prepareUint8List(modelAssetBuffer!); + } + return struct; + } + + @override + List get props => [modelAssetBuffer, modelAssetPath]; +} + +class ClassifierOptions extends Equatable { + const ClassifierOptions({ + this.displayNamesLocale, + this.maxResults, + this.scoreThreshold, + this.categoryAllowlist, + this.categoryDenylist, + }); + + /// The locale to use for display names specified through the TFLite Model + /// Metadata. + final String? displayNamesLocale; + + /// The maximum number of top-scored classification results to return. + final int? maxResults; + + /// Overrides the ones provided in the model metadata. Results below this + /// value are rejected. + final double? scoreThreshold; + + /// Allowlist of category names. If non-empty, classification results whose + /// category name is not in this set will be discarded. Duplicate or unknown + /// category names are ignored. Mutually exclusive with `categoryDenylist`. + final List? categoryAllowlist; + + /// Denylist of category names. If non-empty, classification results whose + /// category name is in this set will be discarded. Duplicate or unknown + /// category names are ignored. Mutually exclusive with `categoryAllowList`. + final List? categoryDenylist; + + Pointer toStruct() { + final struct = calloc(); + _setDisplayNamesLocale(struct.ref); + _setMaxResults(struct.ref); + _setScoreThreshold(struct.ref); + _setAllowlist(struct.ref); + _setDenylist(struct.ref); + return struct; + } + + void _setDisplayNamesLocale(bindings.ClassifierOptions struct) { + if (displayNamesLocale != null) { + struct.display_names_locale = prepareString(displayNamesLocale!); + } + } + + void _setMaxResults(bindings.ClassifierOptions struct) { + // This value must not be zero, and -1 implies no limit. + struct.max_results = maxResults ?? -1; + } + + void _setScoreThreshold(bindings.ClassifierOptions struct) { + if (scoreThreshold != null) { + struct.score_threshold = scoreThreshold!; + } + } + + void _setAllowlist(bindings.ClassifierOptions struct) { + if (categoryAllowlist != null) { + struct.category_allowlist = prepareListOfStrings(categoryAllowlist!); + struct.category_allowlist_count = categoryAllowlist!.length; + } + } + + void _setDenylist(bindings.ClassifierOptions struct) { + if (categoryDenylist != null) { + struct.category_denylist = prepareListOfStrings(categoryDenylist!); + struct.category_denylist_count = categoryDenylist!.length; + } + } + + @override + List get props => [ + displayNamesLocale, + maxResults, + scoreThreshold, + ...(categoryAllowlist ?? []), + ...(categoryDenylist ?? []), + ]; +} diff --git a/packages/mediapipe-core/pubspec.yaml b/packages/mediapipe-core/pubspec.yaml new file mode 100644 index 0000000..4392349 --- /dev/null +++ b/packages/mediapipe-core/pubspec.yaml @@ -0,0 +1,17 @@ +name: mediapipe_core +description: A starting point for Dart libraries or applications. +version: 1.0.0 + +repository: https://github.com/google/flutter-mediapipe/issues +environment: + sdk: ^3.1.0-333.0.dev + +# Add regular dependencies here. +dependencies: + equatable: ^2.0.5 + ffi: ^2.1.0 + +dev_dependencies: + ffigen: ^9.0.1 + lints: ^2.0.0 + test: ^1.21.0 diff --git a/packages/mediapipe-core/test/task_options_test.dart b/packages/mediapipe-core/test/task_options_test.dart new file mode 100644 index 0000000..7721f08 --- /dev/null +++ b/packages/mediapipe-core/test/task_options_test.dart @@ -0,0 +1,116 @@ +import 'dart:ffi'; +import 'dart:typed_data'; + +import 'package:test/test.dart'; +import 'package:mediapipe_core/mediapipe_core.dart'; + +void main() { + group('BaseOptions constructor should', () { + test('enforce exactly one of modelPath and modelBuffer', () { + expect( + () => BaseOptions( + modelAssetPath: 'abc', + modelAssetBuffer: Uint8List.fromList([1, 2, 3]), + ), + throwsA(TypeMatcher()), + ); + + expect(BaseOptions.new, throwsA(TypeMatcher())); + }); + }); + + group('BaseOptions.toStruct/fromStruct should', () { + test('allocate memory in C for a modelAssetPath', () { + final options = BaseOptions(modelAssetPath: 'abc'); + final struct = options.toStruct(); + expect(toDartString(struct.ref.model_asset_path), 'abc'); + + // TODO: Is this the most precise expectation we can make here? + expect(struct.ref.model_asset_buffer, isA>()); + }); + + test('allocate memory in C for a modelAssetBuffer', () { + final options = BaseOptions( + modelAssetBuffer: Uint8List.fromList([1, 2, 3]), + ); + final struct = options.toStruct(); + expect( + toUint8List(struct.ref.model_asset_buffer), + Uint8List.fromList([1, 2, 3]), + ); + + // TODO: Is this the most precise expectation we can make here? + expect(struct.ref.model_asset_path, isA>()); + }); + + test('allocate memory in C for a modelAssetBuffer containing 0', () { + final options = BaseOptions( + modelAssetBuffer: Uint8List.fromList([1, 2, 0, 3]), + ); + final struct = options.toStruct(); + expect( + toUint8List(struct.ref.model_asset_buffer), + Uint8List.fromList([1, 2]), + ); + + // TODO: Is this the most precise expectation we can make here? + expect(struct.ref.model_asset_path, isA>()); + }); + }); + + group('ClassOptions should', () { + test('allocate memory for empty fields', () { + final options = ClassifierOptions(); + + final struct = options.toStruct(); + + expect(struct.ref.max_results, -1); + expect(struct.ref.score_threshold, 0.0); + expect(struct.ref.category_allowlist, isA>()); + expect(struct.ref.category_allowlist_count, 0); + expect(struct.ref.category_denylist, isA>()); + expect(struct.ref.category_denylist_count, 0); + // TODO: Is this the most precise expectation we can make here? + expect(struct.ref.display_names_locale, isA>()); + + // TODO: Could we do something like this? + // (right now, this segfaults) + // expect(struct.ref.display_names_locale.cast().asTypedList(1), + // Uint8List.fromList([0])); + }); + + test('allocate memory for full fields', () { + final options = ClassifierOptions( + displayNamesLocale: 'en', + maxResults: 5, + scoreThreshold: 0.9, + categoryAllowlist: ['good', 'great', 'best'], + categoryDenylist: ['bad', 'terrible', 'worst', 'honestly come on'], + ); + + final struct = options.toStruct(); + + expect(toDartString(struct.ref.display_names_locale), 'en'); + expect(struct.ref.max_results, 5); + expect(struct.ref.score_threshold, greaterThan(0.8999)); + expect(struct.ref.score_threshold, lessThan(0.90001)); + expect( + toDartStrings( + struct.ref.category_allowlist, + struct.ref.category_allowlist_count, + ), + ['good', 'great', 'best'], + ); + expect(struct.ref.category_allowlist_count, 3); + + expect( + toDartStrings( + struct.ref.category_denylist, + struct.ref.category_denylist_count, + ), + ['bad', 'terrible', 'worst', 'honestly come on'], + ); + expect(struct.ref.category_denylist_count, 4); + }); + }); +} diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/category.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/category.h new file mode 100644 index 0000000..b6eede4 --- /dev/null +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/category.h @@ -0,0 +1,50 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CATEGORY_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CATEGORY_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +// Defines a single classification result. +// +// The label maps packed into the TFLite Model Metadata [1] are used to populate +// the 'category_name' and 'display_name' fields. +// +// [1]: https://www.tensorflow.org/lite/convert/metadata +struct Category { + // The index of the category in the classification model output. + int index; + + // The score for this category, e.g. (but not necessarily) a probability in + // [0,1]. + float score; + + // The optional ID for the category, read from the label map packed in the + // TFLite Model Metadata if present. Not necessarily human-readable. + const char* category_name; + + // The optional human-readable name for the category, read from the label map + // packed in the TFLite Model Metadata if present. + const char* display_name; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CATEGORY_H_ diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h new file mode 100644 index 0000000..ef2914e --- /dev/null +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h @@ -0,0 +1,68 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CLASSIFICATION_RESULT_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CLASSIFICATION_RESULT_H_ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Defines classification results for a given classifier head. +struct Classifications { + // The array of predicted categories, usually sorted by descending scores, + // e.g. from high to low probability. + struct Category* categories; + // The number of elements in the categories array. + uint32_t categories_count; + + // The index of the classifier head (i.e. output tensor) these categories + // refer to. This is useful for multi-head models. + int head_index; + + // The optional name of the classifier head, as provided in the TFLite Model + // Metadata [1] if present. This is useful for multi-head models. + // + // [1]: https://www.tensorflow.org/lite/convert/metadata + const char* head_name; +}; + +// Defines classification results of a model. +struct ClassificationResult { + // The classification results for each head of the model. + struct Classifications* classifications; + // The number of classifications in the classifications array. + uint32_t classifications_count; + + // The optional timestamp (in milliseconds) of the start of the chunk of data + // corresponding to these results. + // + // This is only used for classification on time series (e.g. audio + // classification). In these use cases, the amount of data to process might + // exceed the maximum size that the model can process: to solve this, the + // input data is split into multiple chunks starting at different timestamps. + int64_t timestamp_ms; + // Specifies whether the timestamp contains a valid value. + bool has_timestamp_ms; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CLASSIFICATION_RESULT_H_ diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h new file mode 100644 index 0000000..4658fb4 --- /dev/null +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h @@ -0,0 +1,59 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_PROCESSORS_CLASSIFIER_OPTIONS_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_PROCESSORS_CLASSIFIER_OPTIONS_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Classifier options for MediaPipe C classification Tasks. +struct ClassifierOptions { + // The locale to use for display names specified through the TFLite Model + // Metadata, if any. Defaults to English. + char* display_names_locale; + + // The maximum number of top-scored classification results to return. If < 0, + // all available results will be returned. If 0, an invalid argument error is + // returned. + int max_results; + + // Score threshold to override the one provided in the model metadata (if + // any). Results below this value are rejected. + float score_threshold; + + // The allowlist of category names. If non-empty, detection results whose + // category name is not in this set will be filtered out. Duplicate or unknown + // category names are ignored. Mutually exclusive with category_denylist. + char** category_allowlist; + // The number of elements in the category allowlist. + uint32_t category_allowlist_count; + + // The denylist of category names. If non-empty, detection results whose + // category name is in this set will be filtered out. Duplicate or unknown + // category names are ignored. Mutually exclusive with category_allowlist. + char** category_denylist; + // The number of elements in the category denylist. + uint32_t category_denylist_count; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_PROCESSORS_CLASSIFIER_OPTIONS_H_ diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h new file mode 100644 index 0000000..d23b688 --- /dev/null +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h @@ -0,0 +1,36 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_CORE_BASE_OPTIONS_H_ +#define MEDIAPIPE_TASKS_C_CORE_BASE_OPTIONS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +// Base options for MediaPipe C Tasks. +struct BaseOptions { + // The model asset file contents as a string. + char* model_asset_buffer; + + // The path to the model asset to open and mmap in memory. + char* model_asset_path; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_CORE_BASE_OPTIONS_H_ From 688371a53a105516f0da6ba7d3f60a31bfc95e81 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Fri, 13 Oct 2023 10:50:22 -0700 Subject: [PATCH 02/13] adds makefile for all packages --- Makefile | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1b2e7ca --- /dev/null +++ b/Makefile @@ -0,0 +1,40 @@ +# Runs unit tests for `mediapipe_core` +test_core: + cd packages/mediapipe-core && dart test + +# Runs the utility to pull in all header files from `google/mediapipe` +headers: + cd packages/build && dart bin/main.dart headers + +# Runs `ffigen` for `mediapipe_core` +generate_core: + cd packages/mediapipe-core && dart run ffigen --config=ffigen.yaml + +# Runs `ffigen` for `mediapipe_text` +generate_text: + cd packages/mediapipe-task-text && dart run ffigen --config=ffigen.yaml + +# Runs `ffigen` for all packages +generate: generate_core generate_text + +# Compiles the faked C artifacts for testing +compile_fake_text: + # Builds standalone executable + cd packages/mediapipe-task-text/test/c && gcc fake_text_classifier.c -o fake_text_classifier + # Builds what Flutter needs + cd packages/mediapipe-task-text/test/c && gcc -static -c -fPIC *.c -o fake_text_classifier.o + cd packages/mediapipe-task-text/test/c && gcc -shared -o fake_text_classifier.dylib fake_text_classifier.o + +# Runs all text tests +test_text_only: + cd packages/mediapipe-task-text && flutter test + +# Runs `ffigen` for `mediapipe_text` and all text tests +test_text: compile_fake_text test_text_only + +# Runs `ffigen` for all packages, compiles the faked C artifacts, and runs all tests +test: generate_text test_text generate_core test_core + +# Runs `ffigen` for all packages and all tests for all packages +test_only: test_core test_text + From 4a2e4d88c45548e3808bc0bb2a6e210609a010c2 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Mon, 16 Oct 2023 12:53:14 -0700 Subject: [PATCH 03/13] fixes typo in Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1b2e7ca..7d28174 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ test_core: # Runs the utility to pull in all header files from `google/mediapipe` headers: - cd packages/build && dart bin/main.dart headers + cd packages/build_cmd && dart bin/main.dart headers # Runs `ffigen` for `mediapipe_core` generate_core: From 8f0c39c61ccad5f0fd4e4bd3cdde258e6790d138 Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Mon, 16 Oct 2023 18:11:46 -0500 Subject: [PATCH 04/13] Apply suggestions from code review --- packages/mediapipe-core/analysis_options.yaml | 19 +---------------- .../mediapipe-core/lib/src/task_options.dart | 21 ++++++++++++------- 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/packages/mediapipe-core/analysis_options.yaml b/packages/mediapipe-core/analysis_options.yaml index 0e9249f..0b27c8b 100644 --- a/packages/mediapipe-core/analysis_options.yaml +++ b/packages/mediapipe-core/analysis_options.yaml @@ -1,24 +1,7 @@ -# This file configures the static analysis results for your project (errors, -# warnings, and lints). -# -# This enables the 'recommended' set of lints from `package:lints`. -# This set helps identify many issues that may lead to problems when running -# or consuming Dart code, and enforces writing Dart using a single, idiomatic -# style and format. -# -# If you want a smaller set of lints you can change this to specify -# 'package:lints/core.yaml'. These are just the most critical lints -# (the recommended set includes the core lints). -# The core lints are also what is used by pub.dev for scoring packages. +# Specify analysis options. include: package:lints/recommended.yaml -# Uncomment the following section to specify additional rules. - -# linter: -# rules: -# - camel_case_types - analyzer: exclude: - "lib/src/mediapipe_common_bindings.dart" diff --git a/packages/mediapipe-core/lib/src/task_options.dart b/packages/mediapipe-core/lib/src/task_options.dart index 2e1320c..cdcf780 100644 --- a/packages/mediapipe-core/lib/src/task_options.dart +++ b/packages/mediapipe-core/lib/src/task_options.dart @@ -54,18 +54,23 @@ class ClassifierOptions extends Equatable { /// The maximum number of top-scored classification results to return. final int? maxResults; - /// Overrides the ones provided in the model metadata. Results below this - /// value are rejected. + /// Overrides the ones provided in the model metadata. + /// + /// Results below this value are rejected. final double? scoreThreshold; - /// Allowlist of category names. If non-empty, classification results whose - /// category name is not in this set will be discarded. Duplicate or unknown - /// category names are ignored. Mutually exclusive with `categoryDenylist`. + /// Allowlist of category names. + /// + /// If non-empty, classification results whose category name is not in + /// this set will be discarded. Duplicate or unknown category names + /// are ignored. Mutually exclusive with `categoryDenylist`. final List? categoryAllowlist; - /// Denylist of category names. If non-empty, classification results whose - /// category name is in this set will be discarded. Duplicate or unknown - /// category names are ignored. Mutually exclusive with `categoryAllowList`. + /// Denylist of category names. + /// + /// If non-empty, classification results whose category name is in this set + /// will be discarded. Duplicate or unknown category names are ignored. + /// Mutually exclusive with `categoryAllowList`. final List? categoryDenylist; Pointer toStruct() { From 7b1d5ad7b37ca4a4da24fcd5a957a819fcc7bd3e Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Mon, 16 Oct 2023 18:26:43 -0500 Subject: [PATCH 05/13] Update Makefile --- Makefile | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 7d28174..84f0a67 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,32 @@ -# Runs unit tests for `mediapipe_core` -test_core: - cd packages/mediapipe-core && dart test - # Runs the utility to pull in all header files from `google/mediapipe` headers: cd packages/build_cmd && dart bin/main.dart headers +# Runs `ffigen` for all packages +generate: generate_core generate_text + +# Runs `ffigen` for all packages, compiles the faked C artifacts, and runs all tests +test: generate_text test_text generate_core test_core + +# Runs `ffigen` for all packages and all tests for all packages +test_only: test_core test_text + +# Core --- + # Runs `ffigen` for `mediapipe_core` generate_core: cd packages/mediapipe-core && dart run ffigen --config=ffigen.yaml +# Runs unit tests for `mediapipe_core` +test_core: + cd packages/mediapipe-core && dart test + +# Text --- + # Runs `ffigen` for `mediapipe_text` generate_text: cd packages/mediapipe-task-text && dart run ffigen --config=ffigen.yaml -# Runs `ffigen` for all packages -generate: generate_core generate_text - # Compiles the faked C artifacts for testing compile_fake_text: # Builds standalone executable @@ -25,16 +35,9 @@ compile_fake_text: cd packages/mediapipe-task-text/test/c && gcc -static -c -fPIC *.c -o fake_text_classifier.o cd packages/mediapipe-task-text/test/c && gcc -shared -o fake_text_classifier.dylib fake_text_classifier.o -# Runs all text tests -test_text_only: - cd packages/mediapipe-task-text && flutter test - # Runs `ffigen` for `mediapipe_text` and all text tests test_text: compile_fake_text test_text_only -# Runs `ffigen` for all packages, compiles the faked C artifacts, and runs all tests -test: generate_text test_text generate_core test_core - -# Runs `ffigen` for all packages and all tests for all packages -test_only: test_core test_text - +# Runs all text tests +test_text_only: + cd packages/mediapipe-task-text && flutter test From ff83c89f12f5cb5f093b688ead2f995126207e21 Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Mon, 16 Oct 2023 18:48:23 -0500 Subject: [PATCH 06/13] Apply suggestions from code review --- .../tasks/c/components/containers/classification_result.h | 2 ++ .../tasks/c/components/processors/classifier_options.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h index ef2914e..07ba44f 100644 --- a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h @@ -28,6 +28,7 @@ struct Classifications { // The array of predicted categories, usually sorted by descending scores, // e.g. from high to low probability. struct Category* categories; + // The number of elements in the categories array. uint32_t categories_count; @@ -57,6 +58,7 @@ struct ClassificationResult { // exceed the maximum size that the model can process: to solve this, the // input data is split into multiple chunks starting at different timestamps. int64_t timestamp_ms; + // Specifies whether the timestamp contains a valid value. bool has_timestamp_ms; }; diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h index 4658fb4..1c153bd 100644 --- a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h @@ -41,12 +41,14 @@ struct ClassifierOptions { // category name is not in this set will be filtered out. Duplicate or unknown // category names are ignored. Mutually exclusive with category_denylist. char** category_allowlist; + // The number of elements in the category allowlist. uint32_t category_allowlist_count; // The denylist of category names. If non-empty, detection results whose // category name is in this set will be filtered out. Duplicate or unknown // category names are ignored. Mutually exclusive with category_allowlist. + char** category_denylist; // The number of elements in the category denylist. uint32_t category_denylist_count; From a39e75bf150a04480445b4d6a557bd9089b08f1a Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Tue, 17 Oct 2023 12:55:50 -0700 Subject: [PATCH 07/13] updated generated code's location and license (for 3P status) --- packages/mediapipe-core/ffigen.yaml | 20 +++++++++++++++++-- .../lib/generated/core_symbols.yaml | 2 +- .../generated}/mediapipe_common_bindings.dart | 19 ++++++++++++++++-- 3 files changed, 36 insertions(+), 5 deletions(-) rename packages/mediapipe-core/lib/src/{ => third_party/mediapipe/generated}/mediapipe_common_bindings.dart (91%) diff --git a/packages/mediapipe-core/ffigen.yaml b/packages/mediapipe-core/ffigen.yaml index 10c8a88..3f42a17 100644 --- a/packages/mediapipe-core/ffigen.yaml +++ b/packages/mediapipe-core/ffigen.yaml @@ -1,13 +1,29 @@ name: "MediaPipeCommonBindings" description: "Bindings for shared MediaPipe structs common across many tasks" output: - bindings: "lib/src/mediapipe_common_bindings.dart" + bindings: "lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart" symbol-file: output: "package:mediapipe_core/generated/core_symbols.yaml" - import-path: "package:mediapipe_core/src/mediapipe_common_bindings.dart" + import-path: "package:mediapipe_core/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart" headers: entry-points: - "third_party/mediapipe/tasks/c/core/base_options.h" - "third_party/mediapipe/tasks/c/components/containers/category.h" - "third_party/mediapipe/tasks/c/components/containers/classification_result.h" - "third_party/mediapipe/tasks/c/components/processors/classifier_options.h" +preamble: | + /* Copyright 2023 The MediaPipe Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +ffi-native: diff --git a/packages/mediapipe-core/lib/generated/core_symbols.yaml b/packages/mediapipe-core/lib/generated/core_symbols.yaml index 497d7c3..95f3c22 100644 --- a/packages/mediapipe-core/lib/generated/core_symbols.yaml +++ b/packages/mediapipe-core/lib/generated/core_symbols.yaml @@ -1,6 +1,6 @@ format_version: 1.0.0 files: - package:mediapipe_core/src/mediapipe_common_bindings.dart: + package:mediapipe_core/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart: used-config: ffi-native: false symbols: diff --git a/packages/mediapipe-core/lib/src/mediapipe_common_bindings.dart b/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart similarity index 91% rename from packages/mediapipe-core/lib/src/mediapipe_common_bindings.dart rename to packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart index 839b08a..3f87b50 100644 --- a/packages/mediapipe-core/lib/src/mediapipe_common_bindings.dart +++ b/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart @@ -1,3 +1,18 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + // AUTO GENERATED FILE, DO NOT EDIT. // // Generated by `package:ffigen`. @@ -159,12 +174,12 @@ final class ClassifierOptions extends ffi.Struct { external int category_denylist_count; } +const int __bool_true_false_are_defined = 1; + const int true1 = 1; const int false1 = 0; -const int __bool_true_false_are_defined = 1; - const int __WORDSIZE = 64; const int __DARWIN_ONLY_64_BIT_INO_T = 1; From 006e8eff1f65a31461ef23e34400ffab2029cbde Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Tue, 17 Oct 2023 15:52:37 -0700 Subject: [PATCH 08/13] code review responses including: * comments * licensing * resolving testing nits --- packages/mediapipe-core/CHANGELOG.md | 2 +- packages/mediapipe-core/analysis_options.yaml | 7 +- packages/mediapipe-core/ffigen.yaml | 5 +- .../mediapipe-core/lib/mediapipe_core.dart | 6 ++ .../mediapipe-core/lib/src/containers.dart | 55 +++++++++++++- .../mediapipe-core/lib/src/ffi_utils.dart | 30 +++++++- .../mediapipe-core/lib/src/task_options.dart | 43 +++++++++-- .../generated/mediapipe_common_bindings.dart | 76 +++++++++---------- packages/mediapipe-core/pubspec.yaml | 8 +- .../test/task_options_test.dart | 21 +++-- 10 files changed, 186 insertions(+), 67 deletions(-) diff --git a/packages/mediapipe-core/CHANGELOG.md b/packages/mediapipe-core/CHANGELOG.md index effe43c..b78d64c 100644 --- a/packages/mediapipe-core/CHANGELOG.md +++ b/packages/mediapipe-core/CHANGELOG.md @@ -1,3 +1,3 @@ -## 1.0.0 +## 0.0.1 - Initial version. diff --git a/packages/mediapipe-core/analysis_options.yaml b/packages/mediapipe-core/analysis_options.yaml index 0b27c8b..e3e01d6 100644 --- a/packages/mediapipe-core/analysis_options.yaml +++ b/packages/mediapipe-core/analysis_options.yaml @@ -1,7 +1,10 @@ # Specify analysis options. - include: package:lints/recommended.yaml +linter: + rules: + - public_member_api_docs # see https://github.com/flutter/flutter/wiki/Style-guide-for-Flutter-repo#documentation-dartdocs-javadocs-etc + analyzer: exclude: - - "lib/src/mediapipe_common_bindings.dart" + - "**/mediapipe_common_bindings.dart" diff --git a/packages/mediapipe-core/ffigen.yaml b/packages/mediapipe-core/ffigen.yaml index 3f42a17..547b1aa 100644 --- a/packages/mediapipe-core/ffigen.yaml +++ b/packages/mediapipe-core/ffigen.yaml @@ -7,10 +7,7 @@ output: import-path: "package:mediapipe_core/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart" headers: entry-points: - - "third_party/mediapipe/tasks/c/core/base_options.h" - - "third_party/mediapipe/tasks/c/components/containers/category.h" - - "third_party/mediapipe/tasks/c/components/containers/classification_result.h" - - "third_party/mediapipe/tasks/c/components/processors/classifier_options.h" + - "third_party/mediapipe/tasks/c/**" preamble: | /* Copyright 2023 The MediaPipe Authors. diff --git a/packages/mediapipe-core/lib/mediapipe_core.dart b/packages/mediapipe-core/lib/mediapipe_core.dart index 2e62b18..7bb29ef 100644 --- a/packages/mediapipe-core/lib/mediapipe_core.dart +++ b/packages/mediapipe-core/lib/mediapipe_core.dart @@ -1,3 +1,9 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/// Package containing core dependencies for MediaPipe's text, vision, and +/// audio-based tasks. library; export 'src/containers.dart' show Category, Classifications; diff --git a/packages/mediapipe-core/lib/src/containers.dart b/packages/mediapipe-core/lib/src/containers.dart index f8f66e4..fbd7e0e 100644 --- a/packages/mediapipe-core/lib/src/containers.dart +++ b/packages/mediapipe-core/lib/src/containers.dart @@ -1,20 +1,45 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + import 'dart:ffi'; import 'package:ffi/ffi.dart'; import 'package:mediapipe_core/mediapipe_core.dart'; -import 'mediapipe_common_bindings.dart' as bindings; +import 'third_party/mediapipe/generated/mediapipe_common_bindings.dart' + as bindings; +/// Dart representation of MediaPipe's "Category" concept. +/// +/// Category is a util class, that contains a category name, its display name, +/// a float value as score, and the index of the label in the corresponding +/// label file. Typically it's used as result of classification or detection +/// tasks. +/// +/// See more: +/// * [MediaPipe's Category documentation](https://developers.google.com/mediapipe/api/solutions/java/com/google/mediapipe/tasks/components/containers/Category) class Category { + /// Generative constructor that creates a [Category] instance. const Category({ required this.index, required this.score, required this.categoryName, required this.displayName, }); + + /// The index of the label in the corresponding label file. final int index; + + /// The probability score of this label category. final double score; + + /// The label of this category object. final String? categoryName; + + /// The display name of the label, which may be translated for different locales. final String? displayName; + /// Accepts a pointer to a list of structs, and a count representing the length + /// of the list, and returns a list of pure-Dart [Category] instances. static List fromStructs( Pointer structs, int count, @@ -26,6 +51,7 @@ class Category { return categories; } + /// Accepts a pointer to a single struct and returns a pure-Dart [Category] instance. static Category fromStruct(bindings.Category struct) { return Category( index: struct.index, @@ -35,6 +61,9 @@ class Category { ); } + /// Releases all C memory associated with a list of [bindings.Category] pointers. + /// This method is important to call after calling [Category.fromStructs] to + /// convert that C memory into pure-Dart objects. static void freeStructs(Pointer structs, int count) { int index = 0; while (index < count) { @@ -51,16 +80,34 @@ class Category { 'categoryName=$categoryName, displayName=$displayName)'; } +/// Dart representation of MediaPipe's "Classifications" concept. +/// +/// Represents the list of classification for a given classifier head. +/// Typically used as a result for classification tasks. +/// +/// See also: +/// * [MediaPipe's Classifications documentation](https://developers.google.com/mediapipe/api/solutions/java/com/google/mediapipe/tasks/components/containers/Classifications) class Classifications { + /// Generative constructor that creates a [Classifications] instance. const Classifications({ required this.categories, required this.headIndex, required this.headName, }); + + /// A list of Category objects which contain the actual classification + /// information, including human-readable labels and probability scores. final List categories; + + /// The index of the classifier head these entries refer to. final int headIndex; + + /// The optional name of the classifier head, which is the corresponding + /// tensor metadata name. final String? headName; + /// Accepts a pointer to a list of structs, and a count representing the length + /// of the list, and returns a list of pure-Dart [Classifications] instances. static List fromStructs( Pointer structs, int count, @@ -72,6 +119,8 @@ class Classifications { return classifications; } + /// Accepts a pointer to a single struct and returns a pure-Dart [Classifications] + /// instance. static Classifications fromStruct(bindings.Classifications struct) { return Classifications( categories: Category.fromStructs( @@ -83,6 +132,9 @@ class Classifications { ); } + /// Releases all C memory associated with a list of [bindings.Classifications] + /// pointers. This method is important to call after calling [Classifications.fromStructs] + /// to convert that C memory into pure-Dart objects. static void freeStructs( Pointer structs, int count, @@ -97,6 +149,7 @@ class Classifications { calloc.free(structs); } + /// Convenience getter for the first [Category] out of the [categories] list. Category? get firstCategory => categories.isNotEmpty ? categories.first : null; diff --git a/packages/mediapipe-core/lib/src/ffi_utils.dart b/packages/mediapipe-core/lib/src/ffi_utils.dart index 99327d8..a5411c9 100644 --- a/packages/mediapipe-core/lib/src/ffi_utils.dart +++ b/packages/mediapipe-core/lib/src/ffi_utils.dart @@ -1,22 +1,44 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + import 'dart:ffi'; import 'dart:typed_data'; import 'package:ffi/ffi.dart'; +/// Converts a list of Dart strings into their C memory equivalent. +/// +/// See also: +/// * [prepareString] Pointer> prepareListOfStrings(List values) { final ptrArray = calloc>(values.length); for (var i = 0; i < values.length; i++) { - ptrArray[i] = values[i].toNativeUtf8().cast(); + ptrArray[i] = prepareString(values[i]); } return ptrArray; } +/// Converts a single Dart string into its C memory equivalent. +/// +/// See also: +/// * [prepareListOfStrings] Pointer prepareString(String val) => val.toNativeUtf8().cast(); +/// Converts the C memory representation of a string into a Dart [String]. If the +/// supplied pointer is a null pointer, this function returns `null`. +/// +/// See also: +/// * [toDartStrings] String? toDartString(Pointer val) { - if (val == nullptr) return null; + if (val.address == 0) return null; return val.cast().toDartString(); } +/// Converts a list of C memory representations of strings into a list of Dart +/// [String]s. +/// +/// See also: +/// * [toDartString] List toDartStrings(Pointer> val, int length) { final dartStrings = []; int counter = 0; @@ -27,12 +49,16 @@ List toDartStrings(Pointer> val, int length) { return dartStrings; } +/// Converts Dart's representation for binary data, a [Uint8List], into its C +/// memory representation. Pointer prepareUint8List(Uint8List ints) { final Pointer ptr = calloc(ints.length); ptr.asTypedList(ints.length).setAll(0, ints); return ptr.cast(); } +/// Converts a pointer to binary data in C memory to Dart's representation for +/// binary data, a [Uint8List]. Uint8List toUint8List(Pointer val, {int? length}) { final codeUnits = val.cast(); if (length != null) { diff --git a/packages/mediapipe-core/lib/src/task_options.dart b/packages/mediapipe-core/lib/src/task_options.dart index cdcf780..18dd028 100644 --- a/packages/mediapipe-core/lib/src/task_options.dart +++ b/packages/mediapipe-core/lib/src/task_options.dart @@ -1,11 +1,26 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + import 'dart:ffi'; import 'dart:typed_data'; import 'package:equatable/equatable.dart'; import 'package:ffi/ffi.dart'; -import 'mediapipe_common_bindings.dart' as bindings; import 'ffi_utils.dart'; - +import 'third_party/mediapipe/generated/mediapipe_common_bindings.dart' + as bindings; + +/// Dart representation of MediaPipe's "BaseOptions" concept. +/// +/// Used to configure various classifiers by specifying the model they will use +/// for computation. +/// +/// See also: +/// * [MediaPipe's BaseOptions documentation](https://developers.google.com/mediapipe/api/solutions/java/com/google/mediapipe/tasks/core/BaseOptions) +/// * [ClassifierOptions], which is often used in conjunction to specify a +/// classifier's desired behavior. class BaseOptions extends Equatable { + /// Generative constructor that creates a [BaseOptions] instance. const BaseOptions({this.modelAssetBuffer, this.modelAssetPath}) : assert( !(modelAssetBuffer == null && modelAssetPath == null), @@ -22,6 +37,8 @@ class BaseOptions extends Equatable { /// Path to the model asset file. final String? modelAssetPath; + /// Converts this pure-Dart representation into C-memory suitable for the + /// MediaPipe SDK to instantiate various classifiers. Pointer toStruct() { final struct = calloc(); @@ -38,7 +55,16 @@ class BaseOptions extends Equatable { List get props => [modelAssetBuffer, modelAssetPath]; } +/// Dart representation of MediaPipe's "ClassifierOptions" concept. +/// +/// Classifier options shared across MediaPipe classification tasks. +/// +/// See also: +/// * [MediaPipe's ClassifierOptions documentation](https://developers.google.com/mediapipe/api/solutions/java/com/google/mediapipe/tasks/components/processors/ClassifierOptions) +/// * [BaseOptions], which is often used in conjunction to specify a +/// classifier's desired behavior. class ClassifierOptions extends Equatable { + /// Generative constructor that creates a [ClassifierOptions] instance. const ClassifierOptions({ this.displayNamesLocale, this.maxResults, @@ -54,9 +80,8 @@ class ClassifierOptions extends Equatable { /// The maximum number of top-scored classification results to return. final int? maxResults; - /// Overrides the ones provided in the model metadata. - /// - /// Results below this value are rejected. + /// If set, establishes a minimum `score` and leads to the rejection of any + /// categories with lower `score` values. final double? scoreThreshold; /// Allowlist of category names. @@ -64,6 +89,9 @@ class ClassifierOptions extends Equatable { /// If non-empty, classification results whose category name is not in /// this set will be discarded. Duplicate or unknown category names /// are ignored. Mutually exclusive with `categoryDenylist`. + /// + /// See also: + /// * [Category.categoryName] final List? categoryAllowlist; /// Denylist of category names. @@ -71,8 +99,13 @@ class ClassifierOptions extends Equatable { /// If non-empty, classification results whose category name is in this set /// will be discarded. Duplicate or unknown category names are ignored. /// Mutually exclusive with `categoryAllowList`. + /// + /// See also: + /// * [Category.categoryName] final List? categoryDenylist; + /// Converts this pure-Dart representation into C-memory suitable for the + /// MediaPipe SDK to instantiate various classifiers. Pointer toStruct() { final struct = calloc(); _setDisplayNamesLocale(struct.ref); diff --git a/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart b/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart index 3f87b50..13bf3d2 100644 --- a/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart +++ b/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart @@ -25,18 +25,6 @@ final class BaseOptions extends ffi.Struct { external ffi.Pointer model_asset_path; } -final class Category extends ffi.Struct { - @ffi.Int() - external int index; - - @ffi.Float() - external double score; - - external ffi.Pointer category_name; - - external ffi.Pointer display_name; -} - final class __mbstate_t extends ffi.Union { @ffi.Array.multi([128]) external ffi.Array __mbstate8; @@ -129,6 +117,38 @@ final class _opaque_pthread_t extends ffi.Struct { external ffi.Array __opaque; } +final class ClassifierOptions extends ffi.Struct { + external ffi.Pointer display_names_locale; + + @ffi.Int() + external int max_results; + + @ffi.Float() + external double score_threshold; + + external ffi.Pointer> category_allowlist; + + @ffi.Uint32() + external int category_allowlist_count; + + external ffi.Pointer> category_denylist; + + @ffi.Uint32() + external int category_denylist_count; +} + +final class Category extends ffi.Struct { + @ffi.Int() + external int index; + + @ffi.Float() + external double score; + + external ffi.Pointer category_name; + + external ffi.Pointer display_name; +} + final class Classifications extends ffi.Struct { external ffi.Pointer categories; @@ -154,32 +174,6 @@ final class ClassificationResult extends ffi.Struct { external bool has_timestamp_ms; } -final class ClassifierOptions extends ffi.Struct { - external ffi.Pointer display_names_locale; - - @ffi.Int() - external int max_results; - - @ffi.Float() - external double score_threshold; - - external ffi.Pointer> category_allowlist; - - @ffi.Uint32() - external int category_allowlist_count; - - external ffi.Pointer> category_denylist; - - @ffi.Uint32() - external int category_denylist_count; -} - -const int __bool_true_false_are_defined = 1; - -const int true1 = 1; - -const int false1 = 0; - const int __WORDSIZE = 64; const int __DARWIN_ONLY_64_BIT_INO_T = 1; @@ -345,3 +339,9 @@ const int WINT_MAX = 2147483647; const int SIG_ATOMIC_MIN = -2147483648; const int SIG_ATOMIC_MAX = 2147483647; + +const int __bool_true_false_are_defined = 1; + +const int true1 = 1; + +const int false1 = 0; diff --git a/packages/mediapipe-core/pubspec.yaml b/packages/mediapipe-core/pubspec.yaml index 4392349..fbbf44f 100644 --- a/packages/mediapipe-core/pubspec.yaml +++ b/packages/mediapipe-core/pubspec.yaml @@ -1,8 +1,10 @@ name: mediapipe_core -description: A starting point for Dart libraries or applications. -version: 1.0.0 +description: Shared logic and utilities required by other MediaPipe Task packages. +version: 0.0.1-wip + +repository: https://github.com/google/flutter-mediapipe +issue_tracker: https://github.com/google/flutter-mediapipe/issues -repository: https://github.com/google/flutter-mediapipe/issues environment: sdk: ^3.1.0-333.0.dev diff --git a/packages/mediapipe-core/test/task_options_test.dart b/packages/mediapipe-core/test/task_options_test.dart index 7721f08..61ad88f 100644 --- a/packages/mediapipe-core/test/task_options_test.dart +++ b/packages/mediapipe-core/test/task_options_test.dart @@ -1,3 +1,7 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + import 'dart:ffi'; import 'dart:typed_data'; @@ -24,9 +28,7 @@ void main() { final options = BaseOptions(modelAssetPath: 'abc'); final struct = options.toStruct(); expect(toDartString(struct.ref.model_asset_path), 'abc'); - - // TODO: Is this the most precise expectation we can make here? - expect(struct.ref.model_asset_buffer, isA>()); + expectNullPtr(struct.ref.model_asset_buffer); }); test('allocate memory in C for a modelAssetBuffer', () { @@ -38,9 +40,7 @@ void main() { toUint8List(struct.ref.model_asset_buffer), Uint8List.fromList([1, 2, 3]), ); - - // TODO: Is this the most precise expectation we can make here? - expect(struct.ref.model_asset_path, isA>()); + expectNullPtr(struct.ref.model_asset_path); }); test('allocate memory in C for a modelAssetBuffer containing 0', () { @@ -52,9 +52,7 @@ void main() { toUint8List(struct.ref.model_asset_buffer), Uint8List.fromList([1, 2]), ); - - // TODO: Is this the most precise expectation we can make here? - expect(struct.ref.model_asset_path, isA>()); + expectNullPtr(struct.ref.model_asset_path); }); }); @@ -70,8 +68,7 @@ void main() { expect(struct.ref.category_allowlist_count, 0); expect(struct.ref.category_denylist, isA>()); expect(struct.ref.category_denylist_count, 0); - // TODO: Is this the most precise expectation we can make here? - expect(struct.ref.display_names_locale, isA>()); + expectNullPtr(struct.ref.display_names_locale); // TODO: Could we do something like this? // (right now, this segfaults) @@ -114,3 +111,5 @@ void main() { }); }); } + +void expectNullPtr(Pointer ptr) => expect(ptr.address, equals(0)); From 37ea84d8d91dc2ebfd2534728bfcd8b5aa8b6f35 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Wed, 18 Oct 2023 08:26:37 -0700 Subject: [PATCH 09/13] updated README --- packages/mediapipe-core/README.md | 44 +++++++++++-------------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/packages/mediapipe-core/README.md b/packages/mediapipe-core/README.md index 8b55e73..124460d 100644 --- a/packages/mediapipe-core/README.md +++ b/packages/mediapipe-core/README.md @@ -1,39 +1,27 @@ - - -TODO: Put a short description of the package here that helps potential users -know whether this package might be useful for them. - -## Features +A Flutter plugin to use the MediaPipe Core API, which enables multiple Mediapipe tasks. -TODO: List what your package can do. Maybe include images, gifs, or videos. +To learn more about MediaPipe, please visit the [MediaPipe website](https://developers.google.com/mediapipe) -## Getting started +## Getting Started -TODO: List prerequisites and provide or point to information on how to -start using the package. +To get started with MediaPipe, please [see the documentation](https://developers.google.com/mediapipe/solutions/guide). + + +## Issues and feedback -```dart -const like = 'sample'; -``` +Please file Flutter-MediaPipe specific issues, bugs, or feature requests in our [issue tracker](https://github.com/google/flutter-mediapipe/issues/new). -## Additional information +Issues that are specific to Flutter can be filed in the [Flutter issue tracker](https://github.com/flutter/flutter/issues/new). -TODO: Tell users more about the package: where to find more information, how to -contribute to the package, how to file issues, what response they can expect -from the package authors, and more. +To contribute a change to this plugin, +please review our [contribution guide](https://github.com/google/flutter-mediapipe/blob/master/CONTRIBUTING.md) +and open a [pull request](https://github.com/google/flutter-mediapipe/pulls). \ No newline at end of file From 61dbdacb149323e4e64520343248e18cdf3247ff Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Wed, 18 Oct 2023 15:32:16 -0700 Subject: [PATCH 10/13] setup sharing of base analysis_options --- packages/analysis_options.yaml | 6 ++++++ packages/mediapipe-core/analysis_options.yaml | 7 +------ 2 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 packages/analysis_options.yaml diff --git a/packages/analysis_options.yaml b/packages/analysis_options.yaml new file mode 100644 index 0000000..c4d43fe --- /dev/null +++ b/packages/analysis_options.yaml @@ -0,0 +1,6 @@ +include: package:lints/recommended.yaml + +linter: + rules: + - public_member_api_docs # see https://github.com/flutter/flutter/wiki/Style-guide-for-Flutter-repo#documentation-dartdocs-javadocs-etc + diff --git a/packages/mediapipe-core/analysis_options.yaml b/packages/mediapipe-core/analysis_options.yaml index e3e01d6..f2aa915 100644 --- a/packages/mediapipe-core/analysis_options.yaml +++ b/packages/mediapipe-core/analysis_options.yaml @@ -1,9 +1,4 @@ -# Specify analysis options. -include: package:lints/recommended.yaml - -linter: - rules: - - public_member_api_docs # see https://github.com/flutter/flutter/wiki/Style-guide-for-Flutter-repo#documentation-dartdocs-javadocs-etc +include: ../analysis_options.yaml analyzer: exclude: From 6c51cecc87b908e7f3992c5b8f945fdb80ab9502 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Thu, 19 Oct 2023 14:33:02 -0700 Subject: [PATCH 11/13] flutter create --- packages/mediapipe-task-vision/.gitignore | 29 ++++++++++ packages/mediapipe-task-vision/.metadata | 10 ++++ packages/mediapipe-task-vision/CHANGELOG.md | 3 ++ .../analysis_options.yaml | 4 ++ .../lib/mediapipe_vision.dart | 7 +++ packages/mediapipe-task-vision/pubspec.yaml | 54 +++++++++++++++++++ .../test/mediapipe_vision_test.dart | 12 +++++ 7 files changed, 119 insertions(+) create mode 100644 packages/mediapipe-task-vision/.gitignore create mode 100644 packages/mediapipe-task-vision/.metadata create mode 100644 packages/mediapipe-task-vision/CHANGELOG.md create mode 100644 packages/mediapipe-task-vision/analysis_options.yaml create mode 100644 packages/mediapipe-task-vision/lib/mediapipe_vision.dart create mode 100644 packages/mediapipe-task-vision/pubspec.yaml create mode 100644 packages/mediapipe-task-vision/test/mediapipe_vision_test.dart diff --git a/packages/mediapipe-task-vision/.gitignore b/packages/mediapipe-task-vision/.gitignore new file mode 100644 index 0000000..ac5aa98 --- /dev/null +++ b/packages/mediapipe-task-vision/.gitignore @@ -0,0 +1,29 @@ +# Miscellaneous +*.class +*.log +*.pyc +*.swp +.DS_Store +.atom/ +.buildlog/ +.history +.svn/ +migrate_working_dir/ + +# IntelliJ related +*.iml +*.ipr +*.iws +.idea/ + +# The .vscode folder contains launch configuration and tasks you configure in +# VS Code which you may wish to be included in version control, so this line +# is commented out by default. +#.vscode/ + +# Flutter/Dart/Pub related +# Libraries should not include pubspec.lock, per https://dart.dev/guides/libraries/private-files#pubspeclock. +/pubspec.lock +**/doc/api/ +.dart_tool/ +build/ diff --git a/packages/mediapipe-task-vision/.metadata b/packages/mediapipe-task-vision/.metadata new file mode 100644 index 0000000..a4e117e --- /dev/null +++ b/packages/mediapipe-task-vision/.metadata @@ -0,0 +1,10 @@ +# This file tracks properties of this Flutter project. +# Used by Flutter tool to assess capabilities and perform upgrades etc. +# +# This file should be version controlled and should not be manually edited. + +version: + revision: "7cfd3d47fc2e9a9266795e512d03ffd6f7f451fc" + channel: "master" + +project_type: package diff --git a/packages/mediapipe-task-vision/CHANGELOG.md b/packages/mediapipe-task-vision/CHANGELOG.md new file mode 100644 index 0000000..41cc7d8 --- /dev/null +++ b/packages/mediapipe-task-vision/CHANGELOG.md @@ -0,0 +1,3 @@ +## 0.0.1 + +* TODO: Describe initial release. diff --git a/packages/mediapipe-task-vision/analysis_options.yaml b/packages/mediapipe-task-vision/analysis_options.yaml new file mode 100644 index 0000000..a5744c1 --- /dev/null +++ b/packages/mediapipe-task-vision/analysis_options.yaml @@ -0,0 +1,4 @@ +include: package:flutter_lints/flutter.yaml + +# Additional information about this file can be found at +# https://dart.dev/guides/language/analysis-options diff --git a/packages/mediapipe-task-vision/lib/mediapipe_vision.dart b/packages/mediapipe-task-vision/lib/mediapipe_vision.dart new file mode 100644 index 0000000..4e22f4d --- /dev/null +++ b/packages/mediapipe-task-vision/lib/mediapipe_vision.dart @@ -0,0 +1,7 @@ +library mediapipe_vision; + +/// A Calculator. +class Calculator { + /// Returns [value] plus 1. + int addOne(int value) => value + 1; +} diff --git a/packages/mediapipe-task-vision/pubspec.yaml b/packages/mediapipe-task-vision/pubspec.yaml new file mode 100644 index 0000000..f97b100 --- /dev/null +++ b/packages/mediapipe-task-vision/pubspec.yaml @@ -0,0 +1,54 @@ +name: mediapipe_vision +description: "A new Flutter package project." +version: 0.0.1 +homepage: + +environment: + sdk: '>=3.3.0-33.0.dev <4.0.0' + flutter: ">=1.17.0" + +dependencies: + flutter: + sdk: flutter + +dev_dependencies: + flutter_test: + sdk: flutter + flutter_lints: ^2.0.0 + +# For information on the generic Dart part of this file, see the +# following page: https://dart.dev/tools/pub/pubspec + +# The following section is specific to Flutter packages. +flutter: + + # To add assets to your package, add an assets section, like this: + # assets: + # - images/a_dot_burr.jpeg + # - images/a_dot_ham.jpeg + # + # For details regarding assets in packages, see + # https://flutter.dev/assets-and-images/#from-packages + # + # An image asset can refer to one or more resolution-specific "variants", see + # https://flutter.dev/assets-and-images/#resolution-aware + + # To add custom fonts to your package, add a fonts section here, + # in this "flutter" section. Each entry in this list should have a + # "family" key with the font family name, and a "fonts" key with a + # list giving the asset and other descriptors for the font. For + # example: + # fonts: + # - family: Schyler + # fonts: + # - asset: fonts/Schyler-Regular.ttf + # - asset: fonts/Schyler-Italic.ttf + # style: italic + # - family: Trajan Pro + # fonts: + # - asset: fonts/TrajanPro.ttf + # - asset: fonts/TrajanPro_Bold.ttf + # weight: 700 + # + # For details regarding fonts in packages, see + # https://flutter.dev/custom-fonts/#from-packages diff --git a/packages/mediapipe-task-vision/test/mediapipe_vision_test.dart b/packages/mediapipe-task-vision/test/mediapipe_vision_test.dart new file mode 100644 index 0000000..eb4aed8 --- /dev/null +++ b/packages/mediapipe-task-vision/test/mediapipe_vision_test.dart @@ -0,0 +1,12 @@ +import 'package:flutter_test/flutter_test.dart'; + +import 'package:mediapipe_vision/mediapipe_vision.dart'; + +void main() { + test('adds one to input values', () { + final calculator = Calculator(); + expect(calculator.addOne(2), 3); + expect(calculator.addOne(-7), -6); + expect(calculator.addOne(0), 1); + }); +} From c390a8f725239f8a232bd45b2b251d1bdfb35d6f Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Fri, 20 Oct 2023 15:59:55 -0700 Subject: [PATCH 12/13] initial commit of vision task --- .../mediapipe-core/lib/src/ffi_utils.dart | 19 +- .../analysis_options.yaml | 7 +- packages/mediapipe-task-vision/ffigen.yaml | 23 ++ .../lib/mediapipe_vision.dart | 8 +- .../containers/containers.dart | 2 + .../containers/image_classifier_options.dart | 64 ++++ .../containers/image_classifier_result.dart | 67 ++++ .../image_classification.dart | 2 + .../image_classifier.dart | 18 ++ .../image_classifier_io.dart | 73 +++++ .../universal_image_classifier.dart | 16 + .../mediapipe/mediapipe_vision_bindings.dart | 298 ++++++++++++++++++ packages/mediapipe-task-vision/pubspec.yaml | 76 ++--- .../test/mediapipe_vision_test.dart | 11 +- .../tasks/c/vision/image_classifier.h | 135 ++++++++ 15 files changed, 764 insertions(+), 55 deletions(-) create mode 100644 packages/mediapipe-task-vision/ffigen.yaml create mode 100644 packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/containers.dart create mode 100644 packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/image_classifier_options.dart create mode 100644 packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/image_classifier_result.dart create mode 100644 packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classification.dart create mode 100644 packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classifier.dart create mode 100644 packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classifier_io.dart create mode 100644 packages/mediapipe-task-vision/lib/src/tasks/image_classification/universal_image_classifier.dart create mode 100644 packages/mediapipe-task-vision/lib/third_party/mediapipe/mediapipe_vision_bindings.dart create mode 100644 packages/mediapipe-task-vision/third_party/mediapipe/tasks/c/vision/image_classifier.h diff --git a/packages/mediapipe-core/lib/src/ffi_utils.dart b/packages/mediapipe-core/lib/src/ffi_utils.dart index a5411c9..9410e13 100644 --- a/packages/mediapipe-core/lib/src/ffi_utils.dart +++ b/packages/mediapipe-core/lib/src/ffi_utils.dart @@ -39,7 +39,23 @@ String? toDartString(Pointer val) { /// /// See also: /// * [toDartString] -List toDartStrings(Pointer> val, int length) { +List toDartStrings(Pointer> val, [int? length]) => + length != null + ? _toDartStringsWithCount(val, length) + : _toStartStringsUntilNull(val); + +List _toStartStringsUntilNull(Pointer> val) { + final dartStrings = []; + int counter = 0; + while (true) { + if (val[counter].address == 0) break; + dartStrings.add(toDartString(val[counter])); + counter++; + } + return dartStrings; +} + +List _toDartStringsWithCount(Pointer> val, int length) { final dartStrings = []; int counter = 0; while (counter < length) { @@ -69,6 +85,7 @@ Uint8List toUint8List(Pointer val, {int? length}) { return codeUnits.asTypedList(length); } +// Counts the non-null bytes in a string to determine its length int _length(Pointer codeUnits) { var length = 0; while (codeUnits[length] != 0) { diff --git a/packages/mediapipe-task-vision/analysis_options.yaml b/packages/mediapipe-task-vision/analysis_options.yaml index a5744c1..ee7f274 100644 --- a/packages/mediapipe-task-vision/analysis_options.yaml +++ b/packages/mediapipe-task-vision/analysis_options.yaml @@ -1,4 +1,5 @@ -include: package:flutter_lints/flutter.yaml +include: ../analysis_options.yaml -# Additional information about this file can be found at -# https://dart.dev/guides/language/analysis-options +analyzer: + exclude: + - "**/mediapipe_vision_bindings.dart" diff --git a/packages/mediapipe-task-vision/ffigen.yaml b/packages/mediapipe-task-vision/ffigen.yaml new file mode 100644 index 0000000..309cbe0 --- /dev/null +++ b/packages/mediapipe-task-vision/ffigen.yaml @@ -0,0 +1,23 @@ +name: "MediaPipeVisionBindings" +description: "Bindings for MediaPipe structs for vision-related tasks" +output: + bindings: "lib/third_party/mediapipe/mediapipe_vision_bindings.dart" +headers: + entry-points: + - "third_party/mediapipe/tasks/c/**" +import: + symbol-files: + - "package:mediapipe_core/generated/core_symbols.yaml" +preamble: | + /* Copyright 2023 The MediaPipe Authors. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +ffi-native: diff --git a/packages/mediapipe-task-vision/lib/mediapipe_vision.dart b/packages/mediapipe-task-vision/lib/mediapipe_vision.dart index 4e22f4d..4efb967 100644 --- a/packages/mediapipe-task-vision/lib/mediapipe_vision.dart +++ b/packages/mediapipe-task-vision/lib/mediapipe_vision.dart @@ -1,7 +1 @@ -library mediapipe_vision; - -/// A Calculator. -class Calculator { - /// Returns [value] plus 1. - int addOne(int value) => value + 1; -} +export 'src/tasks/image_classification/image_classification.dart'; diff --git a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/containers.dart b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/containers.dart new file mode 100644 index 0000000..6b4e1d6 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/containers.dart @@ -0,0 +1,2 @@ +export 'image_classifier_options.dart'; +export 'image_classifier_result.dart'; diff --git a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/image_classifier_options.dart b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/image_classifier_options.dart new file mode 100644 index 0000000..dd52267 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/image_classifier_options.dart @@ -0,0 +1,64 @@ +import 'dart:ffi'; + +import 'package:ffi/ffi.dart'; +import 'package:flutter/foundation.dart'; +import 'package:mediapipe_core/mediapipe_core.dart'; +import '../../../../third_party/mediapipe/mediapipe_vision_bindings.dart' + as bindings; + +/// Configuration object for a MediaPipe image classifier. +/// +/// See also: +/// * [MediaPipe's ImageClassifierOptions documentation](https://developers.google.com/mediapipe/api/solutions/js/tasks-vision.imageclassifieroptions) +class ImageClassifierOptions { + /// Generative constructor which creates an [ImageClassifierOptions] instance. + const ImageClassifierOptions({ + required this.baseOptions, + this.classifierOptions = const ClassifierOptions(), + }); + + /// Convenience constructor that looks for the model asset at the given file + /// system location. + /// + /// Note: Because this constructor assumes access to the file system, it is + /// not suitable for Flutter Web projects. + factory ImageClassifierOptions.fromAssetPath( + String assetPath, { + ClassifierOptions classifierOptions = const ClassifierOptions(), + }) { + assert(!kIsWeb, 'fromAssetPath cannot be used on the web'); + return ImageClassifierOptions( + baseOptions: BaseOptions(modelAssetPath: assetPath), + classifierOptions: classifierOptions, + ); + } + + /// Convenience constructor that uses a model existing in memory. + /// + /// Note: This constructor can be used on any target platform. + factory ImageClassifierOptions.fromAssetBuffer( + Uint8List assetBuffer, { + ClassifierOptions classifierOptions = const ClassifierOptions(), + }) => + ImageClassifierOptions( + baseOptions: BaseOptions(modelAssetBuffer: assetBuffer), + classifierOptions: classifierOptions, + ); + + /// Container storing options universal to all MediaPipe tasks (namely, which + /// model to use for the task). + final BaseOptions baseOptions; + + /// Container storing options universal to all MediaPipe classifiers. + final ClassifierOptions classifierOptions; + + /// Converts this [ImageClassifierOptions] instance into its C representation. + Pointer toStruct() { + final struct = calloc(); + + struct.ref.base_options = baseOptions.toStruct().ref; + struct.ref.classifier_options = classifierOptions.toStruct().ref; + + return struct; + } +} diff --git a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/image_classifier_result.dart b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/image_classifier_result.dart new file mode 100644 index 0000000..df19187 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/image_classifier_result.dart @@ -0,0 +1,67 @@ +import 'dart:ffi'; +import 'package:ffi/ffi.dart'; +import 'package:mediapipe_core/mediapipe_core.dart'; + +import '../../../../third_party/mediapipe/mediapipe_vision_bindings.dart' + as bindings; + +/// Container with results of MediaPipe's `classifyImage` task. +/// +/// See also: +/// * [MediaPipe's ImageClassifierResult documentation](https://developers.google.com/mediapipe/api/solutions/js/tasks-vision.imageclassifierresult) +class ImageClassifierResult { + /// Generative constructor which creates an [ImageClassifierResult] instance. + const ImageClassifierResult({ + required this.classifications, + this.timestamp, + }); + + /// The classification results for each head of the model. + final List classifications; + + /// The optional timestamp (as a Duration) of the start of the chunk of data + /// corresponding to these results. + /// + /// This is only used for classification on time series (e.g. audio + /// classification). In these use cases, the amount of data to process might + /// exceed the maximum size that the model can process: to solve this, the + /// input data is split into multiple chunks starting at different timestamps. + final Duration? timestamp; + + /// Factory constructor which converts the C representation of an + /// ImageClassifierResult into an actual [ImageClassifierResult]. + factory ImageClassifierResult.fromStruct( + bindings.ImageClassifierResult struct, + ) => + ImageClassifierResult( + classifications: Classifications.fromStructs( + struct.classifications, + struct.classifications_count, + ), + timestamp: struct.has_timestamp_ms + ? Duration(milliseconds: struct.timestamp_ms) + : null, + ); + + /// Deallocates all memory associated with an [bindings.ImageClassifierResult] + /// in C memory. + static void freeStruct(Pointer struct) { + Classifications.freeStructs( + struct.ref.classifications, + struct.ref.classifications_count, + ); + calloc.free(struct); + } + + /// Convenience helper for the first [Classifications] object. + Classifications? get firstClassification => + classifications.isNotEmpty ? classifications.first : null; + + @override + String toString() { + final classificationStrings = + classifications.map((cat) => cat.toString()).join(', '); + return 'TextClassifierResult(classifications=[$classificationStrings], ' + 'timestamp=$timestamp)'; + } +} diff --git a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classification.dart b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classification.dart new file mode 100644 index 0000000..9a10f93 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classification.dart @@ -0,0 +1,2 @@ +export 'containers/containers.dart'; +export 'image_classifier.dart'; diff --git a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classifier.dart b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classifier.dart new file mode 100644 index 0000000..e26d283 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classifier.dart @@ -0,0 +1,18 @@ +import 'dart:ui'; +import 'package:mediapipe_vision/mediapipe_vision.dart'; + +export 'universal_image_classifier.dart' + if (dart.library.html) 'image_classifier_web.dart' + if (dart.library.io) 'image_classifier_io.dart'; + +/// Channel to analyze images via MediaPipe's vision classification task. +abstract class BaseImageClassifier { + /// Generative constructor. + BaseImageClassifier({required this.options}); + + /// Configuration object for tasks completed by this classifier. + final ImageClassifierOptions options; + + /// Sends a `String` value to MediaPipe for classification. + Future classify(Image image); +} diff --git a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classifier_io.dart b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classifier_io.dart new file mode 100644 index 0000000..546983e --- /dev/null +++ b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classifier_io.dart @@ -0,0 +1,73 @@ +import 'dart:ui'; +import 'dart:ffi'; + +import 'package:ffi/ffi.dart'; +import 'package:logging/logging.dart'; +import 'package:mediapipe_core/mediapipe_core.dart'; + +import 'image_classifier.dart'; +import 'containers/containers.dart'; +import '../../../third_party/mediapipe/mediapipe_vision_bindings.dart' + as bindings; + +final _log = Logger('ImageClassifier'); + +/// Channel to analyze images via MediaPipe's vision classification task. +class ImageClassifier extends BaseImageClassifier { + /// Generative constructor which creates an [ImageClassifer] instance. + ImageClassifier({required super.options}); + + @override + Future classify(Image image) async { + final optionsPtr = options.toStruct(); + + // TODO(craiglabenz): What is the correct value to pass to this initializer? + // Maybe it should be 0? + final initError = calloc>(1); + + final classifierPtr = + bindings.image_classifier_create(optionsPtr, initError); + _log.finest('classifierPtr: $classifierPtr'); + + if (classifierPtr.address == 0) { + throw Exception( + 'Configuration error:\n${_flattenErrors(initError)}', + ); + } + calloc.free(initError); + + // Allocate the container for our results + final resultsPtr = calloc(); + final classifyError = calloc>(1); + + int result = bindings.image_classifier_classify_image( + classifierPtr, + toImageStruct(image), + resultsPtr, + classifyError, + ); + if (result == 0) { + // Failure + throw Exception( + 'Classification error:\n${_flattenErrors(classifyError)}', + ); + } else { + // Success + return ImageClassifierResult.fromStruct(resultsPtr.ref); + } + } + + /// Converts a Dart [Image] into the MediaPipe representation of an image. + Pointer toImageStruct(Image img) { + // TODO(craiglabenz): Implement + throw UnimplementedError('Implement `toImageStruct`'); + } + + String _flattenErrors(Pointer> errors) { + final errorMessages = toDartStrings(errors); + final flattenedError = + errorMessages.where((String? val) => val != null).join('\n'); + calloc.free(errors); + return flattenedError; + } +} diff --git a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/universal_image_classifier.dart b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/universal_image_classifier.dart new file mode 100644 index 0000000..60787c6 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/universal_image_classifier.dart @@ -0,0 +1,16 @@ +import 'dart:ui'; +import 'package:mediapipe_vision/mediapipe_vision.dart'; + +/// Channel to classify an image via MediaPipe's "classifyImage" task. +class ImageClassifier extends BaseImageClassifier { + /// Generative constructor. + ImageClassifier({required super.options}) { + throw Exception('Must use the web or FFI implementations'); + } + + /// Sends an `Image` to MediaPipe for classification. + @override + Future classify(Image image) { + throw Exception('Must use the web or FFI implementations'); + } +} diff --git a/packages/mediapipe-task-vision/lib/third_party/mediapipe/mediapipe_vision_bindings.dart b/packages/mediapipe-task-vision/lib/third_party/mediapipe/mediapipe_vision_bindings.dart new file mode 100644 index 0000000..41a4445 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/third_party/mediapipe/mediapipe_vision_bindings.dart @@ -0,0 +1,298 @@ +/* Copyright 2023 The MediaPipe Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// AUTO GENERATED FILE, DO NOT EDIT. +// +// Generated by `package:ffigen`. +// ignore_for_file: type=lint +import 'dart:ffi' as ffi; +import 'package:mediapipe_core/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart' + as imp1; + +@ffi.Native< + ffi.Pointer Function(ffi.Pointer, + ffi.Pointer>)>(symbol: 'image_classifier_create') +external ffi.Pointer image_classifier_create( + ffi.Pointer options, + ffi.Pointer> error_msg, +); + +@ffi.Native< + ffi.Int Function( + ffi.Pointer, + ffi.Pointer, + ffi.Pointer, + ffi.Pointer>)>( + symbol: 'image_classifier_classify_image') +external int image_classifier_classify_image( + ffi.Pointer classifier, + ffi.Pointer image, + ffi.Pointer result, + ffi.Pointer> error_msg, +); + +@ffi.Native)>( + symbol: 'image_classifier_close_result') +external void image_classifier_close_result( + ffi.Pointer result, +); + +@ffi.Native< + ffi.Int Function(ffi.Pointer, + ffi.Pointer>)>(symbol: 'image_classifier_close') +external int image_classifier_close( + ffi.Pointer classifier, + ffi.Pointer> error_msg, +); + +abstract class ImageFormat { + static const int UNKNOWN = 0; + static const int SRGB = 1; + static const int SRGBA = 2; + static const int GRAY8 = 3; + static const int SBGRA = 11; +} + +abstract class RunningMode { + static const int IMAGE = 1; + static const int VIDEO = 2; + static const int LIVE_STREAM = 3; +} + +final class ImageFrame extends ffi.Struct { + @ffi.Int32() + external int format; + + external ffi.Pointer image_buffer; + + @ffi.Int() + external int width; + + @ffi.Int() + external int height; +} + +final class GpuBuffer extends ffi.Struct { + @ffi.Int() + external int height; + + @ffi.Int() + external int width; +} + +final class MpImage extends ffi.Struct { + @ffi.Int32() + external int type; + + external UnnamedUnion1 unnamed; +} + +final class UnnamedUnion1 extends ffi.Union { + external ImageFrame image_frame; + + external GpuBuffer gpu_buffer; +} + +final class ImageClassifierOptions extends ffi.Struct { + external imp1.BaseOptions base_options; + + @ffi.Int32() + external int running_mode; + + external imp1.ClassifierOptions classifier_options; + + external ffi.Pointer< + ffi.NativeFunction< + ffi.Void Function(ffi.Pointer, + ffi.Pointer, ffi.Int64)>> result_callback_fn; + + @ffi.Int() + external int result_callback; +} + +typedef ImageClassifierResult = imp1.ClassificationResult; + +const int IMAGE_FRAME = 0; + +const int GPU_BUFFER = 1; + +const int __bool_true_false_are_defined = 1; + +const int true1 = 1; + +const int false1 = 0; + +const int __WORDSIZE = 64; + +const int __DARWIN_ONLY_64_BIT_INO_T = 1; + +const int __DARWIN_ONLY_UNIX_CONFORMANCE = 1; + +const int __DARWIN_ONLY_VERS_1050 = 1; + +const int __DARWIN_UNIX03 = 1; + +const int __DARWIN_64_BIT_INO_T = 1; + +const int __DARWIN_VERS_1050 = 1; + +const int __DARWIN_NON_CANCELABLE = 0; + +const String __DARWIN_SUF_EXTSN = '\$DARWIN_EXTSN'; + +const int __DARWIN_C_ANSI = 4096; + +const int __DARWIN_C_FULL = 900000; + +const int __DARWIN_C_LEVEL = 900000; + +const int __STDC_WANT_LIB_EXT1__ = 1; + +const int __DARWIN_NO_LONG_LONG = 0; + +const int _DARWIN_FEATURE_64_BIT_INODE = 1; + +const int _DARWIN_FEATURE_ONLY_64_BIT_INODE = 1; + +const int _DARWIN_FEATURE_ONLY_VERS_1050 = 1; + +const int _DARWIN_FEATURE_ONLY_UNIX_CONFORMANCE = 1; + +const int _DARWIN_FEATURE_UNIX_CONFORMANCE = 3; + +const int __has_ptrcheck = 0; + +const int __DARWIN_NULL = 0; + +const int __PTHREAD_SIZE__ = 8176; + +const int __PTHREAD_ATTR_SIZE__ = 56; + +const int __PTHREAD_MUTEXATTR_SIZE__ = 8; + +const int __PTHREAD_MUTEX_SIZE__ = 56; + +const int __PTHREAD_CONDATTR_SIZE__ = 8; + +const int __PTHREAD_COND_SIZE__ = 40; + +const int __PTHREAD_ONCE_SIZE__ = 8; + +const int __PTHREAD_RWLOCK_SIZE__ = 192; + +const int __PTHREAD_RWLOCKATTR_SIZE__ = 16; + +const int USER_ADDR_NULL = 0; + +const int INT8_MAX = 127; + +const int INT16_MAX = 32767; + +const int INT32_MAX = 2147483647; + +const int INT64_MAX = 9223372036854775807; + +const int INT8_MIN = -128; + +const int INT16_MIN = -32768; + +const int INT32_MIN = -2147483648; + +const int INT64_MIN = -9223372036854775808; + +const int UINT8_MAX = 255; + +const int UINT16_MAX = 65535; + +const int UINT32_MAX = 4294967295; + +const int UINT64_MAX = -1; + +const int INT_LEAST8_MIN = -128; + +const int INT_LEAST16_MIN = -32768; + +const int INT_LEAST32_MIN = -2147483648; + +const int INT_LEAST64_MIN = -9223372036854775808; + +const int INT_LEAST8_MAX = 127; + +const int INT_LEAST16_MAX = 32767; + +const int INT_LEAST32_MAX = 2147483647; + +const int INT_LEAST64_MAX = 9223372036854775807; + +const int UINT_LEAST8_MAX = 255; + +const int UINT_LEAST16_MAX = 65535; + +const int UINT_LEAST32_MAX = 4294967295; + +const int UINT_LEAST64_MAX = -1; + +const int INT_FAST8_MIN = -128; + +const int INT_FAST16_MIN = -32768; + +const int INT_FAST32_MIN = -2147483648; + +const int INT_FAST64_MIN = -9223372036854775808; + +const int INT_FAST8_MAX = 127; + +const int INT_FAST16_MAX = 32767; + +const int INT_FAST32_MAX = 2147483647; + +const int INT_FAST64_MAX = 9223372036854775807; + +const int UINT_FAST8_MAX = 255; + +const int UINT_FAST16_MAX = 65535; + +const int UINT_FAST32_MAX = 4294967295; + +const int UINT_FAST64_MAX = -1; + +const int INTPTR_MAX = 9223372036854775807; + +const int INTPTR_MIN = -9223372036854775808; + +const int UINTPTR_MAX = -1; + +const int INTMAX_MAX = 9223372036854775807; + +const int UINTMAX_MAX = -1; + +const int INTMAX_MIN = -9223372036854775808; + +const int PTRDIFF_MIN = -9223372036854775808; + +const int PTRDIFF_MAX = 9223372036854775807; + +const int SIZE_MAX = -1; + +const int RSIZE_MAX = 9223372036854775807; + +const int WCHAR_MAX = 2147483647; + +const int WCHAR_MIN = -2147483648; + +const int WINT_MIN = -2147483648; + +const int WINT_MAX = 2147483647; + +const int SIG_ATOMIC_MIN = -2147483648; + +const int SIG_ATOMIC_MAX = 2147483647; diff --git a/packages/mediapipe-task-vision/pubspec.yaml b/packages/mediapipe-task-vision/pubspec.yaml index f97b100..57da1cc 100644 --- a/packages/mediapipe-task-vision/pubspec.yaml +++ b/packages/mediapipe-task-vision/pubspec.yaml @@ -1,54 +1,58 @@ name: mediapipe_vision description: "A new Flutter package project." version: 0.0.1 -homepage: +repository: https://github.com/google/flutter-mediapipe +issue_tracker: https://github.com/google/flutter-mediapipe/issues +publish_to: none environment: - sdk: '>=3.3.0-33.0.dev <4.0.0' + sdk: ">=3.3.0-33.0.dev <4.0.0" flutter: ">=1.17.0" dependencies: + ffi: ^2.1.0 flutter: sdk: flutter + logging: ^1.2.0 + mediapipe_core: + path: ../mediapipe-core dev_dependencies: + ffigen: ^9.0.1 + flutter_lints: ^2.0.0 flutter_test: sdk: flutter - flutter_lints: ^2.0.0 # For information on the generic Dart part of this file, see the # following page: https://dart.dev/tools/pub/pubspec - # The following section is specific to Flutter packages. -flutter: - - # To add assets to your package, add an assets section, like this: - # assets: - # - images/a_dot_burr.jpeg - # - images/a_dot_ham.jpeg - # - # For details regarding assets in packages, see - # https://flutter.dev/assets-and-images/#from-packages - # - # An image asset can refer to one or more resolution-specific "variants", see - # https://flutter.dev/assets-and-images/#resolution-aware - - # To add custom fonts to your package, add a fonts section here, - # in this "flutter" section. Each entry in this list should have a - # "family" key with the font family name, and a "fonts" key with a - # list giving the asset and other descriptors for the font. For - # example: - # fonts: - # - family: Schyler - # fonts: - # - asset: fonts/Schyler-Regular.ttf - # - asset: fonts/Schyler-Italic.ttf - # style: italic - # - family: Trajan Pro - # fonts: - # - asset: fonts/TrajanPro.ttf - # - asset: fonts/TrajanPro_Bold.ttf - # weight: 700 - # - # For details regarding fonts in packages, see - # https://flutter.dev/custom-fonts/#from-packages +flutter: null +# To add assets to your package, add an assets section, like this: +# assets: +# - images/a_dot_burr.jpeg +# - images/a_dot_ham.jpeg +# +# For details regarding assets in packages, see +# https://flutter.dev/assets-and-images/#from-packages +# +# An image asset can refer to one or more resolution-specific "variants", see +# https://flutter.dev/assets-and-images/#resolution-aware +# To add custom fonts to your package, add a fonts section here, +# in this "flutter" section. Each entry in this list should have a +# "family" key with the font family name, and a "fonts" key with a +# list giving the asset and other descriptors for the font. For +# example: +# fonts: +# - family: Schyler +# fonts: +# - asset: fonts/Schyler-Regular.ttf +# - asset: fonts/Schyler-Italic.ttf +# style: italic +# - family: Trajan Pro +# fonts: +# - asset: fonts/TrajanPro.ttf +# - asset: fonts/TrajanPro_Bold.ttf +# weight: 700 +# +# For details regarding fonts in packages, see +# https://flutter.dev/custom-fonts/#from-packages diff --git a/packages/mediapipe-task-vision/test/mediapipe_vision_test.dart b/packages/mediapipe-task-vision/test/mediapipe_vision_test.dart index eb4aed8..593cb08 100644 --- a/packages/mediapipe-task-vision/test/mediapipe_vision_test.dart +++ b/packages/mediapipe-task-vision/test/mediapipe_vision_test.dart @@ -1,12 +1,7 @@ -import 'package:flutter_test/flutter_test.dart'; +// import 'package:flutter_test/flutter_test.dart'; -import 'package:mediapipe_vision/mediapipe_vision.dart'; +// import 'package:mediapipe_vision/mediapipe_vision.dart'; void main() { - test('adds one to input values', () { - final calculator = Calculator(); - expect(calculator.addOne(2), 3); - expect(calculator.addOne(-7), -6); - expect(calculator.addOne(0), 1); - }); + // TODO(craiglabenz): Write tests } diff --git a/packages/mediapipe-task-vision/third_party/mediapipe/tasks/c/vision/image_classifier.h b/packages/mediapipe-task-vision/third_party/mediapipe/tasks/c/vision/image_classifier.h new file mode 100644 index 0000000..7e57b7c --- /dev/null +++ b/packages/mediapipe-task-vision/third_party/mediapipe/tasks/c/vision/image_classifier.h @@ -0,0 +1,135 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_VISION_IMAGE_CLASSIFIER_IMAGE_CLASSIFIER_H_ +#define MEDIAPIPE_TASKS_C_VISION_IMAGE_CLASSIFIER_IMAGE_CLASSIFIER_H_ + +#include + +#include "../../../../../../mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h" +#include "../../../../../../mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h" +#include "../../../../../../mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h" + +#ifndef MP_EXPORT +#define MP_EXPORT __attribute__((visibility("default"))) +#endif // MP_EXPORT + +#ifdef __cplusplus +extern "C" { +#endif + + typedef ClassificationResult ImageClassifierResult; + + // Supported image formats. + enum ImageFormat { + UNKNOWN = 0, + SRGB = 1, + SRGBA = 2, + GRAY8 = 3, + SBGRA = 11 // compatible with Flutter `bgra8888` format. + }; + + // Supported processing modes. + enum RunningMode { + IMAGE = 1, + VIDEO = 2, + LIVE_STREAM = 3, + }; + + // Structure to hold image frame. + struct ImageFrame { + enum ImageFormat format; + const uint8_t* image_buffer; + int width; + int height; + }; + + // TODO: Add GPU buffer declaration and proccessing logic for it. + struct GpuBuffer { + int height; + int width; + }; + + // The object to contain an image, realizes `OneOf` concept. + struct MpImage { + enum { IMAGE_FRAME, GPU_BUFFER } type; + union { + struct ImageFrame image_frame; + struct GpuBuffer gpu_buffer; + }; + }; + + // The options for configuring a Mediapipe image classifier task. + struct ImageClassifierOptions { + // Base options for configuring MediaPipe Tasks, such as specifying the model + // file with metadata, accelerator options, op resolver, etc. + struct BaseOptions base_options; + + // The running mode of the task. Default to the image mode. + // Image classifier has three running modes: + // 1) The image mode for classifying image on single image inputs. + // 2) The video mode for classifying image on the decoded frames of a video. + // 3) The live stream mode for classifying image on the live stream of input + // data, such as from camera. In this mode, the "result_callback" below must + // be specified to receive the segmentation results asynchronously. + RunningMode running_mode; + + // Options for configuring the classifier behavior, such as score threshold, + // number of results, etc. + struct ClassifierOptions classifier_options; + + // The user-defined result callback for processing live stream data. + // The result callback should only be specified when the running mode is set + // to RunningMode::LIVE_STREAM. + typedef void (*result_callback_fn)(ImageClassifierResult*, const MpImage*, + int64_t); + result_callback_fn result_callback; + }; + + // Creates an ImageClassifier from provided `options`. + // Returns a pointer to the image classifier on success. + // If an error occurs, returns `nullptr` and sets the error parameter to an + // an error message (if `error_msg` is not nullptr). You must free the memory + // allocated for the error message. + MP_EXPORT void* image_classifier_create(struct ImageClassifierOptions* options, + char** error_msg = nullptr); + + // Performs image classification on the input `image`. Returns `0` on success. + // If an error occurs, returns an error code and sets the error parameter to an + // an error message (if `error_msg` is not nullptr). You must free the memory + // allocated for the error message. + // + // TODO: Add API for video and live stream processing. + MP_EXPORT int image_classifier_classify_image(void* classifier, + const MpImage* image, + ImageClassifierResult* result, + char** error_msg = nullptr); + + // Frees the memory allocated inside a ImageClassifierResult result. + // Does not free the result pointer itself. + MP_EXPORT void image_classifier_close_result(ImageClassifierResult* result); + + // Frees image classifier. + // If an error occurs, returns an error code and sets the error parameter to an + // an error message (if `error_msg` is not nullptr). You must free the memory + // allocated for the error message. + MP_EXPORT int image_classifier_close(void* classifier, + char** error_msg = nullptr); + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_VISION_IMAGE_CLASSIFIER_IMAGE_CLASSIFIER_H_ From 79ca52e0a1e0c6abfe93e1b9a6064256f223723a Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Tue, 4 Jun 2024 12:58:09 -0700 Subject: [PATCH 13/13] Updates structure of mediapipe_vision package --- .../mediapipe-task-vision/lib/interface.dart | 5 ++ packages/mediapipe-task-vision/lib/io.dart | 5 ++ .../lib/mediapipe_vision.dart | 11 ++- .../lib/src/interface/interface.dart | 4 + .../lib/src/interface/tasks/tasks.dart | 3 + .../lib/src/io/mediapipe_vision.dart | 4 + .../lib/src/io/tasks/tasks.dart | 3 + .../containers/containers.dart | 2 - .../containers/image_classifier_options.dart | 64 ---------------- .../containers/image_classifier_result.dart | 67 ----------------- .../image_classification.dart | 2 - .../image_classifier.dart | 18 ----- .../image_classifier_io.dart | 73 ------------------- .../universal_image_classifier.dart | 16 ---- .../lib/universal_mediapipe_vision.dart | 3 + 15 files changed, 37 insertions(+), 243 deletions(-) create mode 100644 packages/mediapipe-task-vision/lib/interface.dart create mode 100644 packages/mediapipe-task-vision/lib/io.dart create mode 100644 packages/mediapipe-task-vision/lib/src/interface/interface.dart create mode 100644 packages/mediapipe-task-vision/lib/src/interface/tasks/tasks.dart create mode 100644 packages/mediapipe-task-vision/lib/src/io/mediapipe_vision.dart create mode 100644 packages/mediapipe-task-vision/lib/src/io/tasks/tasks.dart delete mode 100644 packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/containers.dart delete mode 100644 packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/image_classifier_options.dart delete mode 100644 packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/image_classifier_result.dart delete mode 100644 packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classification.dart delete mode 100644 packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classifier.dart delete mode 100644 packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classifier_io.dart delete mode 100644 packages/mediapipe-task-vision/lib/src/tasks/image_classification/universal_image_classifier.dart create mode 100644 packages/mediapipe-task-vision/lib/universal_mediapipe_vision.dart diff --git a/packages/mediapipe-task-vision/lib/interface.dart b/packages/mediapipe-task-vision/lib/interface.dart new file mode 100644 index 0000000..110d7e3 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/interface.dart @@ -0,0 +1,5 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +export 'src/interface/interface.dart'; \ No newline at end of file diff --git a/packages/mediapipe-task-vision/lib/io.dart b/packages/mediapipe-task-vision/lib/io.dart new file mode 100644 index 0000000..5a1b3ae --- /dev/null +++ b/packages/mediapipe-task-vision/lib/io.dart @@ -0,0 +1,5 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +export 'src/io/mediapipe_vision.dart'; diff --git a/packages/mediapipe-task-vision/lib/mediapipe_vision.dart b/packages/mediapipe-task-vision/lib/mediapipe_vision.dart index 4efb967..d85d255 100644 --- a/packages/mediapipe-task-vision/lib/mediapipe_vision.dart +++ b/packages/mediapipe-task-vision/lib/mediapipe_vision.dart @@ -1 +1,10 @@ -export 'src/tasks/image_classification/image_classification.dart'; +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/// Package containing MediaPipe's vision-specific tasks. +library mediapipe_vision; + +export 'universal_mediapipe_vision.dart' + if (dart.library.html) 'src/web/mediapipe_vision.dart' + if (dart.library.io) 'src/io/mediapipe_vision.dart'; diff --git a/packages/mediapipe-task-vision/lib/src/interface/interface.dart b/packages/mediapipe-task-vision/lib/src/interface/interface.dart new file mode 100644 index 0000000..70736a9 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/src/interface/interface.dart @@ -0,0 +1,4 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + diff --git a/packages/mediapipe-task-vision/lib/src/interface/tasks/tasks.dart b/packages/mediapipe-task-vision/lib/src/interface/tasks/tasks.dart new file mode 100644 index 0000000..854d12e --- /dev/null +++ b/packages/mediapipe-task-vision/lib/src/interface/tasks/tasks.dart @@ -0,0 +1,3 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. diff --git a/packages/mediapipe-task-vision/lib/src/io/mediapipe_vision.dart b/packages/mediapipe-task-vision/lib/src/io/mediapipe_vision.dart new file mode 100644 index 0000000..70736a9 --- /dev/null +++ b/packages/mediapipe-task-vision/lib/src/io/mediapipe_vision.dart @@ -0,0 +1,4 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + diff --git a/packages/mediapipe-task-vision/lib/src/io/tasks/tasks.dart b/packages/mediapipe-task-vision/lib/src/io/tasks/tasks.dart new file mode 100644 index 0000000..854d12e --- /dev/null +++ b/packages/mediapipe-task-vision/lib/src/io/tasks/tasks.dart @@ -0,0 +1,3 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. diff --git a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/containers.dart b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/containers.dart deleted file mode 100644 index 6b4e1d6..0000000 --- a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/containers.dart +++ /dev/null @@ -1,2 +0,0 @@ -export 'image_classifier_options.dart'; -export 'image_classifier_result.dart'; diff --git a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/image_classifier_options.dart b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/image_classifier_options.dart deleted file mode 100644 index dd52267..0000000 --- a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/image_classifier_options.dart +++ /dev/null @@ -1,64 +0,0 @@ -import 'dart:ffi'; - -import 'package:ffi/ffi.dart'; -import 'package:flutter/foundation.dart'; -import 'package:mediapipe_core/mediapipe_core.dart'; -import '../../../../third_party/mediapipe/mediapipe_vision_bindings.dart' - as bindings; - -/// Configuration object for a MediaPipe image classifier. -/// -/// See also: -/// * [MediaPipe's ImageClassifierOptions documentation](https://developers.google.com/mediapipe/api/solutions/js/tasks-vision.imageclassifieroptions) -class ImageClassifierOptions { - /// Generative constructor which creates an [ImageClassifierOptions] instance. - const ImageClassifierOptions({ - required this.baseOptions, - this.classifierOptions = const ClassifierOptions(), - }); - - /// Convenience constructor that looks for the model asset at the given file - /// system location. - /// - /// Note: Because this constructor assumes access to the file system, it is - /// not suitable for Flutter Web projects. - factory ImageClassifierOptions.fromAssetPath( - String assetPath, { - ClassifierOptions classifierOptions = const ClassifierOptions(), - }) { - assert(!kIsWeb, 'fromAssetPath cannot be used on the web'); - return ImageClassifierOptions( - baseOptions: BaseOptions(modelAssetPath: assetPath), - classifierOptions: classifierOptions, - ); - } - - /// Convenience constructor that uses a model existing in memory. - /// - /// Note: This constructor can be used on any target platform. - factory ImageClassifierOptions.fromAssetBuffer( - Uint8List assetBuffer, { - ClassifierOptions classifierOptions = const ClassifierOptions(), - }) => - ImageClassifierOptions( - baseOptions: BaseOptions(modelAssetBuffer: assetBuffer), - classifierOptions: classifierOptions, - ); - - /// Container storing options universal to all MediaPipe tasks (namely, which - /// model to use for the task). - final BaseOptions baseOptions; - - /// Container storing options universal to all MediaPipe classifiers. - final ClassifierOptions classifierOptions; - - /// Converts this [ImageClassifierOptions] instance into its C representation. - Pointer toStruct() { - final struct = calloc(); - - struct.ref.base_options = baseOptions.toStruct().ref; - struct.ref.classifier_options = classifierOptions.toStruct().ref; - - return struct; - } -} diff --git a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/image_classifier_result.dart b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/image_classifier_result.dart deleted file mode 100644 index df19187..0000000 --- a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/containers/image_classifier_result.dart +++ /dev/null @@ -1,67 +0,0 @@ -import 'dart:ffi'; -import 'package:ffi/ffi.dart'; -import 'package:mediapipe_core/mediapipe_core.dart'; - -import '../../../../third_party/mediapipe/mediapipe_vision_bindings.dart' - as bindings; - -/// Container with results of MediaPipe's `classifyImage` task. -/// -/// See also: -/// * [MediaPipe's ImageClassifierResult documentation](https://developers.google.com/mediapipe/api/solutions/js/tasks-vision.imageclassifierresult) -class ImageClassifierResult { - /// Generative constructor which creates an [ImageClassifierResult] instance. - const ImageClassifierResult({ - required this.classifications, - this.timestamp, - }); - - /// The classification results for each head of the model. - final List classifications; - - /// The optional timestamp (as a Duration) of the start of the chunk of data - /// corresponding to these results. - /// - /// This is only used for classification on time series (e.g. audio - /// classification). In these use cases, the amount of data to process might - /// exceed the maximum size that the model can process: to solve this, the - /// input data is split into multiple chunks starting at different timestamps. - final Duration? timestamp; - - /// Factory constructor which converts the C representation of an - /// ImageClassifierResult into an actual [ImageClassifierResult]. - factory ImageClassifierResult.fromStruct( - bindings.ImageClassifierResult struct, - ) => - ImageClassifierResult( - classifications: Classifications.fromStructs( - struct.classifications, - struct.classifications_count, - ), - timestamp: struct.has_timestamp_ms - ? Duration(milliseconds: struct.timestamp_ms) - : null, - ); - - /// Deallocates all memory associated with an [bindings.ImageClassifierResult] - /// in C memory. - static void freeStruct(Pointer struct) { - Classifications.freeStructs( - struct.ref.classifications, - struct.ref.classifications_count, - ); - calloc.free(struct); - } - - /// Convenience helper for the first [Classifications] object. - Classifications? get firstClassification => - classifications.isNotEmpty ? classifications.first : null; - - @override - String toString() { - final classificationStrings = - classifications.map((cat) => cat.toString()).join(', '); - return 'TextClassifierResult(classifications=[$classificationStrings], ' - 'timestamp=$timestamp)'; - } -} diff --git a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classification.dart b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classification.dart deleted file mode 100644 index 9a10f93..0000000 --- a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classification.dart +++ /dev/null @@ -1,2 +0,0 @@ -export 'containers/containers.dart'; -export 'image_classifier.dart'; diff --git a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classifier.dart b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classifier.dart deleted file mode 100644 index e26d283..0000000 --- a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classifier.dart +++ /dev/null @@ -1,18 +0,0 @@ -import 'dart:ui'; -import 'package:mediapipe_vision/mediapipe_vision.dart'; - -export 'universal_image_classifier.dart' - if (dart.library.html) 'image_classifier_web.dart' - if (dart.library.io) 'image_classifier_io.dart'; - -/// Channel to analyze images via MediaPipe's vision classification task. -abstract class BaseImageClassifier { - /// Generative constructor. - BaseImageClassifier({required this.options}); - - /// Configuration object for tasks completed by this classifier. - final ImageClassifierOptions options; - - /// Sends a `String` value to MediaPipe for classification. - Future classify(Image image); -} diff --git a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classifier_io.dart b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classifier_io.dart deleted file mode 100644 index 546983e..0000000 --- a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/image_classifier_io.dart +++ /dev/null @@ -1,73 +0,0 @@ -import 'dart:ui'; -import 'dart:ffi'; - -import 'package:ffi/ffi.dart'; -import 'package:logging/logging.dart'; -import 'package:mediapipe_core/mediapipe_core.dart'; - -import 'image_classifier.dart'; -import 'containers/containers.dart'; -import '../../../third_party/mediapipe/mediapipe_vision_bindings.dart' - as bindings; - -final _log = Logger('ImageClassifier'); - -/// Channel to analyze images via MediaPipe's vision classification task. -class ImageClassifier extends BaseImageClassifier { - /// Generative constructor which creates an [ImageClassifer] instance. - ImageClassifier({required super.options}); - - @override - Future classify(Image image) async { - final optionsPtr = options.toStruct(); - - // TODO(craiglabenz): What is the correct value to pass to this initializer? - // Maybe it should be 0? - final initError = calloc>(1); - - final classifierPtr = - bindings.image_classifier_create(optionsPtr, initError); - _log.finest('classifierPtr: $classifierPtr'); - - if (classifierPtr.address == 0) { - throw Exception( - 'Configuration error:\n${_flattenErrors(initError)}', - ); - } - calloc.free(initError); - - // Allocate the container for our results - final resultsPtr = calloc(); - final classifyError = calloc>(1); - - int result = bindings.image_classifier_classify_image( - classifierPtr, - toImageStruct(image), - resultsPtr, - classifyError, - ); - if (result == 0) { - // Failure - throw Exception( - 'Classification error:\n${_flattenErrors(classifyError)}', - ); - } else { - // Success - return ImageClassifierResult.fromStruct(resultsPtr.ref); - } - } - - /// Converts a Dart [Image] into the MediaPipe representation of an image. - Pointer toImageStruct(Image img) { - // TODO(craiglabenz): Implement - throw UnimplementedError('Implement `toImageStruct`'); - } - - String _flattenErrors(Pointer> errors) { - final errorMessages = toDartStrings(errors); - final flattenedError = - errorMessages.where((String? val) => val != null).join('\n'); - calloc.free(errors); - return flattenedError; - } -} diff --git a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/universal_image_classifier.dart b/packages/mediapipe-task-vision/lib/src/tasks/image_classification/universal_image_classifier.dart deleted file mode 100644 index 60787c6..0000000 --- a/packages/mediapipe-task-vision/lib/src/tasks/image_classification/universal_image_classifier.dart +++ /dev/null @@ -1,16 +0,0 @@ -import 'dart:ui'; -import 'package:mediapipe_vision/mediapipe_vision.dart'; - -/// Channel to classify an image via MediaPipe's "classifyImage" task. -class ImageClassifier extends BaseImageClassifier { - /// Generative constructor. - ImageClassifier({required super.options}) { - throw Exception('Must use the web or FFI implementations'); - } - - /// Sends an `Image` to MediaPipe for classification. - @override - Future classify(Image image) { - throw Exception('Must use the web or FFI implementations'); - } -} diff --git a/packages/mediapipe-task-vision/lib/universal_mediapipe_vision.dart b/packages/mediapipe-task-vision/lib/universal_mediapipe_vision.dart new file mode 100644 index 0000000..854d12e --- /dev/null +++ b/packages/mediapipe-task-vision/lib/universal_mediapipe_vision.dart @@ -0,0 +1,3 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file.