Adds mediapipe_text package

.github/workflows/main.yaml

@@ -7,7 +7,7 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main, ffi-wrapper, ffi-wrapper-core-pkg]
+    branches: [main, ffi-wrapper, ffi-wrapper-text-pkg]
   workflow_dispatch:
   schedule:
     - cron: "0 0 * * *" # Every day at midnight
@@ -25,7 +25,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        flutter_version: [stable, beta, master]
+        flutter_version: [master]
+        # TODO(craiglabenz): reverse once CI works
+        # flutter_version: [stable, beta, master]
         # TODO(craiglabenz): Add `ubuntu-latest` and `windows-latest` when those artifacts exist
         os: [macos-latest]
     steps:

.vscode/settings.json

@@ -0,0 +1,8 @@
+{
+    "files.associations": {
+        "base_options.h": "c",
+        "__config": "c",
+        "text_classifier.h": "c",
+        "category.h": "c"
+    }
+}

Makefile

@@ -1,16 +1,30 @@
 # Runs the utility to pull in all header files from `google/mediapipe`
 headers:
-	cd packages/build_cmd && dart bin/main.dart headers
+	cd tool/builder && dart bin/main.dart headers
 
 # Runs `ffigen` for all packages
 generate: generate_core generate_text
 
-# Runs `ffigen` for all packages, compiles the faked C artifacts, and runs all tests
+# Runs `ffigen` for all packages and runs all tests
 test: generate_text test_text generate_core test_core
 
-# Runs `ffigen` for all packages and all tests for all packages
+# Runs all tests for all packages
 test_only: test_core test_text
 
+# Runs `sdks_finder` to update manifest files
+sdks:
+	dart tool/builder/bin/main.dart sdks
+
+# Rebuilds the MediaPipe task for macOS
+# Assumes google/mediapipe and google/flutter-mediapipe are siblings on the file system
+compile_text_classifier_macos_arm:
+	cd ../mediapipe && bazel build --linkopt -s --config darwin_arm64 --strip always --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/tasks/c/text/text_classifier:libtext_classifier.dylib
+	cd ../mediapipe && sudo cp bazel-bin/mediapipe/tasks/c/text/text_classifier/libtext_classifier.dylib ../flutter-mediapipe/packages/mediapipe-task-text/example/assets/libtext_classifier_arm64.dylib
+
+compile_text_classifier_macos_x86:
+	cd ../mediapipe && bazel build --linkopt -s --config darwin_x86_64 --strip always --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/tasks/c/text/text_classifier:libtext_classifier.dylib
+	cd ../mediapipe && sudo cp bazel-bin/mediapipe/tasks/c/text/text_classifier/libtext_classifier.dylib ../flutter-mediapipe/packages/mediapipe-task-text/example/assets/libtext_classifier_x64.dylib
+
 # Runs `sdks_finder` to update manifest files
 sdks:
 	dart tool/builder/bin/main.dart sdks
@@ -29,19 +43,12 @@ test_core:
 
 # Runs `ffigen` for `mediapipe_text`
 generate_text:
-	cd packages/mediapipe-task-text && dart run ffigen --config=ffigen.yaml
-
-# Compiles the faked C artifacts for testing
-compile_fake_text:
-	# Builds standalone executable
-	cd packages/mediapipe-task-text/test/c && gcc fake_text_classifier.c -o fake_text_classifier
-	# Builds what Flutter needs
-	cd packages/mediapipe-task-text/test/c && gcc -static -c -fPIC *.c -o fake_text_classifier.o
-	cd packages/mediapipe-task-text/test/c && gcc -shared -o fake_text_classifier.dylib fake_text_classifier.o 
-
-# Runs `ffigen` for `mediapipe_text` and all text tests
-test_text: compile_fake_text test_text_only
+	cd packages/mediapipe-task-text && dart --enable-experiment=native-assets run ffigen --config=ffigen.yaml
 
 # Runs all text tests
-test_text_only:
+test_text:
 	cd packages/mediapipe-task-text && flutter test
+	cd packages/mediapipe-task-text/example && flutter test
+
+example_text:
+	cd packages/mediapipe-task-text/example && flutter run -d macos

packages/mediapipe-core/README.md

@@ -24,4 +24,4 @@ Issues that are specific to Flutter can be filed in the [Flutter issue tracker](
 
 To contribute a change to this plugin,
 please review our [contribution guide](https://github.com/google/flutter-mediapipe/blob/master/CONTRIBUTING.md)
-and open a [pull request](https://github.com/google/flutter-mediapipe/pulls).
+and open a [pull request](https://github.com/google/flutter-mediapipe/pulls).

packages/mediapipe-core/analysis_options.yaml

@@ -1,5 +1,9 @@
 include: ../analysis_options.yaml
 
+linter:
+  rules:
+    - public_member_api_docs # see https://github.com/flutter/flutter/wiki/Style-guide-for-Flutter-repo#documentation-dartdocs-javadocs-etc
+
 analyzer:
   exclude:
     - "**/mediapipe_common_bindings.dart"

packages/mediapipe-core/ffigen.yaml

@@ -7,7 +7,7 @@ output:
     import-path: "package:mediapipe_core/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart"
 headers:
   entry-points:
-    - "third_party/mediapipe/tasks/c/**"
+    - "third_party/mediapipe/tasks/c/**.h"
 preamble: |
   /* Copyright 2023 The MediaPipe Authors.
 

packages/mediapipe-core/lib/generated/core_symbols.yaml

@@ -6,6 +6,8 @@ files:
     symbols:
       c:@S@BaseOptions:
         name: BaseOptions
+      c:@S@Categories:
+        name: Categories
       c:@S@Category:
         name: Category
       c:@S@ClassificationResult:

packages/mediapipe-core/lib/mediapipe_core.dart

@@ -6,6 +6,11 @@
 /// audio-based tasks.
 library;
 
-export 'src/containers.dart' show Category, Classifications;
+export 'src/containers.dart'
+    show
+        Category,
+        Classifications,
+        ClassifierResult,
+        TimestampedClassifierResult;
 export 'src/ffi_utils.dart';
 export 'src/task_options.dart' show BaseOptions, ClassifierOptions;

packages/mediapipe-core/lib/src/containers.dart

@@ -40,19 +40,19 @@ class Category {
 
   /// Accepts a pointer to a list of structs, and a count representing the length
   /// of the list, and returns a list of pure-Dart [Category] instances.
-  static List<Category> fromStructs(
+  static List<Category> structsToDart(
     Pointer<bindings.Category> structs,
     int count,
   ) {
     final categories = <Category>[];
     for (int i = 0; i < count; i++) {
-      categories.add(fromStruct(structs[i]));
+      categories.add(structToDart(structs[i]));
     }
     return categories;
   }
 
   /// Accepts a pointer to a single struct and returns a pure-Dart [Category] instance.
-  static Category fromStruct(bindings.Category struct) {
+  static Category structToDart(bindings.Category struct) {
     return Category(
       index: struct.index,
       score: struct.score,
@@ -62,19 +62,25 @@ class Category {
   }
 
   /// Releases all C memory associated with a list of [bindings.Category] pointers.
-  /// This method is important to call after calling [Category.fromStructs] to
+  /// This method is important to call after calling [Category.structsToDart] to
   /// convert that C memory into pure-Dart objects.
   static void freeStructs(Pointer<bindings.Category> structs, int count) {
     int index = 0;
     while (index < count) {
       bindings.Category obj = structs[index];
-      calloc.free(obj.category_name);
-      calloc.free(obj.display_name);
+      Category.freeStruct(obj);
       index++;
     }
     calloc.free(structs);
   }
 
+  /// Releases the string fields associated with a single [bindings.Category]
+  /// struct.
+  static void freeStruct(bindings.Category struct) {
+    calloc.free(struct.category_name);
+    calloc.free(struct.display_name);
+  }
+
   @override
   String toString() => 'Category(index=$index, score=$score, '
       'categoryName=$categoryName, displayName=$displayName)';
@@ -108,22 +114,22 @@ class Classifications {
 
   /// Accepts a pointer to a list of structs, and a count representing the length
   /// of the list, and returns a list of pure-Dart [Classifications] instances.
-  static List<Classifications> fromStructs(
+  static List<Classifications> structsToDart(
     Pointer<bindings.Classifications> structs,
     int count,
   ) {
     final classifications = <Classifications>[];
     for (int i = 0; i < count; i++) {
-      classifications.add(fromStruct(structs[i]));
+      classifications.add(structToDart(structs[i]));
     }
     return classifications;
   }
 
   /// Accepts a pointer to a single struct and returns a pure-Dart [Classifications]
   /// instance.
-  static Classifications fromStruct(bindings.Classifications struct) {
+  static Classifications structToDart(bindings.Classifications struct) {
     return Classifications(
-      categories: Category.fromStructs(
+      categories: Category.structsToDart(
         struct.categories,
         struct.categories_count,
       ),
@@ -133,7 +139,7 @@ class Classifications {
   }
 
   /// Releases all C memory associated with a list of [bindings.Classifications]
-  /// pointers. This method is important to call after calling [Classifications.fromStructs]
+  /// pointers. This method is important to call after calling [Classifications.structsToDart]
   /// to convert that C memory into pure-Dart objects.
   static void freeStructs(
     Pointer<bindings.Classifications> structs,
@@ -160,3 +166,52 @@ class Classifications {
         'headIndex=$headIndex, headName=$headName)';
   }
 }
+
+/// Container for classification results across any MediaPipe task.
+abstract class ClassifierResult {
+  /// Container for classification results across any MediaPipe task.
+  const ClassifierResult({
+    required this.classifications,
+  });
+
+  /// The classification results for each head of the model.
+  final List<Classifications> classifications;
+
+  /// Convenience helper for the first [Classifications] object.
+  Classifications? get firstClassification =>
+      classifications.isNotEmpty ? classifications.first : null;
+
+  @override
+  String toString() {
+    final classificationStrings =
+        classifications.map((cat) => cat.toString()).join(', ');
+    return '$runtimeType(classifications=[$classificationStrings])';
+  }
+}
+
+/// Container for classification results that may describe a slice of time
+/// within a larger, streaming data source (.e.g, a video or audio file).
+abstract class TimestampedClassifierResult extends ClassifierResult {
+  /// Generative constructor.
+  const TimestampedClassifierResult({
+    required super.classifications,
+    this.timestamp,
+  });
+
+  /// The optional timestamp (as a [Duration]) of the start of the chunk of data
+  /// corresponding to these results.
+  ///
+  /// This is only used for classification on time series (e.g. audio
+  /// classification). In these use cases, the amount of data to process might
+  /// exceed the maximum size that the model can process: to solve this, the
+  /// input data is split into multiple chunks starting at different timestamps.
+  final Duration? timestamp;
+
+  @override
+  String toString() {
+    final classificationStrings =
+        classifications.map((cat) => cat.toString()).join(', ');
+    return '$runtimeType(classifications=[$classificationStrings], '
+        'timestamp=$timestamp)';
+  }
+}

packages/mediapipe-core/lib/src/ffi_utils.dart

@@ -76,3 +76,21 @@ int _length(Pointer<Uint8> codeUnits) {
   }
   return length;
 }
+
+/// Offers convenience and readability extensions for detecting null pointers.
+extension NullAwarePtr on Pointer {
+  /// Returns true if this is a null pointer.
+  bool get isNullPointer => address == 0;
+
+  /// Returns true if this is not a null pointer.
+  bool get isNotNullPointer => address != 0;
+}
+
+/// Returns true if this nullable pointer is both not-null and not a `NullPtr`,
+/// which is to say, its address is 0x00000000.
+bool isNotNullOrNullPointer(Pointer? ptr) =>
+    ptr != null && ptr.isNotNullPointer;
+
+/// Returns true if this nullable pointer is either null OR is a `NullPtr`,
+/// which is to say, its address is 0x00000000.
+bool isNullOrNullPointer(Pointer? ptr) => ptr == null || ptr.isNullPointer;