diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 0000000..45325b8 --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,36 @@ +name: Main Branch CI + +# Declare default permissions as read only. +permissions: read-all + +on: + push: + branches: [main] + pull_request: + branches: [main, ffi-wrapper, ffi-wrapper-core-pkg] + workflow_dispatch: + schedule: + - cron: "0 0 * * *" # Every day at midnight + +defaults: + run: + shell: bash + +jobs: + flutter-tests: + name: Test mediapipe_core against ${{ matrix.flutter_version }} + runs-on: ${{ matrix.os }} + # Skip running job on forks + if: github.repository == 'google/flutter-mediapipe' + strategy: + fail-fast: false + matrix: + flutter_version: [stable, beta, master] + # TODO(craiglabenz): Add `ubuntu-latest` and `windows-latest` when those artifacts exist + os: [macos-latest] + steps: + - uses: actions/checkout@v4 + - uses: subosito/flutter-action@v2 + with: + channel: ${{ matrix.flutter_version }} + - run: ./tool/mediapipe_ci_script_${{ matrix.flutter_version }}.sh diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..84f0a67 --- /dev/null +++ b/Makefile @@ -0,0 +1,43 @@ +# Runs the utility to pull in all header files from `google/mediapipe` +headers: + cd packages/build_cmd && dart bin/main.dart headers + +# Runs `ffigen` for all packages +generate: generate_core generate_text + +# Runs `ffigen` for all packages, compiles the faked C artifacts, and runs all tests +test: generate_text test_text generate_core test_core + +# Runs `ffigen` for all packages and all tests for all packages +test_only: test_core test_text + +# Core --- + +# Runs `ffigen` for `mediapipe_core` +generate_core: + cd packages/mediapipe-core && dart run ffigen --config=ffigen.yaml + +# Runs unit tests for `mediapipe_core` +test_core: + cd packages/mediapipe-core && dart test + +# Text --- + +# Runs `ffigen` for `mediapipe_text` +generate_text: + cd packages/mediapipe-task-text && dart run ffigen --config=ffigen.yaml + +# Compiles the faked C artifacts for testing +compile_fake_text: + # Builds standalone executable + cd packages/mediapipe-task-text/test/c && gcc fake_text_classifier.c -o fake_text_classifier + # Builds what Flutter needs + cd packages/mediapipe-task-text/test/c && gcc -static -c -fPIC *.c -o fake_text_classifier.o + cd packages/mediapipe-task-text/test/c && gcc -shared -o fake_text_classifier.dylib fake_text_classifier.o + +# Runs `ffigen` for `mediapipe_text` and all text tests +test_text: compile_fake_text test_text_only + +# Runs all text tests +test_text_only: + cd packages/mediapipe-task-text && flutter test diff --git a/packages/analysis_options.yaml b/packages/analysis_options.yaml new file mode 100644 index 0000000..c4d43fe --- /dev/null +++ b/packages/analysis_options.yaml @@ -0,0 +1,6 @@ +include: package:lints/recommended.yaml + +linter: + rules: + - public_member_api_docs # see https://github.com/flutter/flutter/wiki/Style-guide-for-Flutter-repo#documentation-dartdocs-javadocs-etc + diff --git a/packages/mediapipe-core/.gitignore b/packages/mediapipe-core/.gitignore new file mode 100644 index 0000000..3cceda5 --- /dev/null +++ b/packages/mediapipe-core/.gitignore @@ -0,0 +1,7 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ + +# Avoid committing pubspec.lock for library packages; see +# https://dart.dev/guides/libraries/private-files#pubspeclock. +pubspec.lock diff --git a/packages/mediapipe-core/CHANGELOG.md b/packages/mediapipe-core/CHANGELOG.md new file mode 100644 index 0000000..b78d64c --- /dev/null +++ b/packages/mediapipe-core/CHANGELOG.md @@ -0,0 +1,3 @@ +## 0.0.1 + +- Initial version. diff --git a/packages/mediapipe-core/README.md b/packages/mediapipe-core/README.md new file mode 100644 index 0000000..124460d --- /dev/null +++ b/packages/mediapipe-core/README.md @@ -0,0 +1,27 @@ +# MediaPipe Core for Flutter + +![pub package](https://img.shields.io/pub/v/mediapipe_core) + +A Flutter plugin to use the MediaPipe Core API, which enables multiple Mediapipe tasks. + +To learn more about MediaPipe, please visit the [MediaPipe website](https://developers.google.com/mediapipe) + +## Getting Started + +To get started with MediaPipe, please [see the documentation](https://developers.google.com/mediapipe/solutions/guide). + + + +## Issues and feedback + +Please file Flutter-MediaPipe specific issues, bugs, or feature requests in our [issue tracker](https://github.com/google/flutter-mediapipe/issues/new). + +Issues that are specific to Flutter can be filed in the [Flutter issue tracker](https://github.com/flutter/flutter/issues/new). + +To contribute a change to this plugin, +please review our [contribution guide](https://github.com/google/flutter-mediapipe/blob/master/CONTRIBUTING.md) +and open a [pull request](https://github.com/google/flutter-mediapipe/pulls). \ No newline at end of file diff --git a/packages/mediapipe-core/analysis_options.yaml b/packages/mediapipe-core/analysis_options.yaml new file mode 100644 index 0000000..f2aa915 --- /dev/null +++ b/packages/mediapipe-core/analysis_options.yaml @@ -0,0 +1,5 @@ +include: ../analysis_options.yaml + +analyzer: + exclude: + - "**/mediapipe_common_bindings.dart" diff --git a/packages/mediapipe-core/ffigen.yaml b/packages/mediapipe-core/ffigen.yaml new file mode 100644 index 0000000..547b1aa --- /dev/null +++ b/packages/mediapipe-core/ffigen.yaml @@ -0,0 +1,26 @@ +name: "MediaPipeCommonBindings" +description: "Bindings for shared MediaPipe structs common across many tasks" +output: + bindings: "lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart" + symbol-file: + output: "package:mediapipe_core/generated/core_symbols.yaml" + import-path: "package:mediapipe_core/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart" +headers: + entry-points: + - "third_party/mediapipe/tasks/c/**" +preamble: | + /* Copyright 2023 The MediaPipe Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +ffi-native: diff --git a/packages/mediapipe-core/lib/generated/core_symbols.yaml b/packages/mediapipe-core/lib/generated/core_symbols.yaml new file mode 100644 index 0000000..95f3c22 --- /dev/null +++ b/packages/mediapipe-core/lib/generated/core_symbols.yaml @@ -0,0 +1,38 @@ +format_version: 1.0.0 +files: + package:mediapipe_core/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart: + used-config: + ffi-native: false + symbols: + c:@S@BaseOptions: + name: BaseOptions + c:@S@Category: + name: Category + c:@S@ClassificationResult: + name: ClassificationResult + c:@S@Classifications: + name: Classifications + c:@S@ClassifierOptions: + name: ClassifierOptions + c:@S@__darwin_pthread_handler_rec: + name: __darwin_pthread_handler_rec + c:@S@_opaque_pthread_attr_t: + name: _opaque_pthread_attr_t + c:@S@_opaque_pthread_cond_t: + name: _opaque_pthread_cond_t + c:@S@_opaque_pthread_condattr_t: + name: _opaque_pthread_condattr_t + c:@S@_opaque_pthread_mutex_t: + name: _opaque_pthread_mutex_t + c:@S@_opaque_pthread_mutexattr_t: + name: _opaque_pthread_mutexattr_t + c:@S@_opaque_pthread_once_t: + name: _opaque_pthread_once_t + c:@S@_opaque_pthread_rwlock_t: + name: _opaque_pthread_rwlock_t + c:@S@_opaque_pthread_rwlockattr_t: + name: _opaque_pthread_rwlockattr_t + c:@S@_opaque_pthread_t: + name: _opaque_pthread_t + c:@UA@__mbstate_t: + name: __mbstate_t diff --git a/packages/mediapipe-core/lib/mediapipe_core.dart b/packages/mediapipe-core/lib/mediapipe_core.dart new file mode 100644 index 0000000..7bb29ef --- /dev/null +++ b/packages/mediapipe-core/lib/mediapipe_core.dart @@ -0,0 +1,11 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/// Package containing core dependencies for MediaPipe's text, vision, and +/// audio-based tasks. +library; + +export 'src/containers.dart' show Category, Classifications; +export 'src/ffi_utils.dart'; +export 'src/task_options.dart' show BaseOptions, ClassifierOptions; diff --git a/packages/mediapipe-core/lib/src/containers.dart b/packages/mediapipe-core/lib/src/containers.dart new file mode 100644 index 0000000..5111040 --- /dev/null +++ b/packages/mediapipe-core/lib/src/containers.dart @@ -0,0 +1,162 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:ffi'; +import 'package:ffi/ffi.dart'; +import 'package:mediapipe_core/mediapipe_core.dart'; +import 'third_party/mediapipe/generated/mediapipe_common_bindings.dart' + as bindings; + +/// Dart representation of MediaPipe's "Category" concept. +/// +/// Category is a util class, that contains a [categoryName], its [displayName], +/// a float value as [score], and the [index] of the label in the corresponding +/// label file. Typically it's used as result of classification or detection +/// tasks. +/// +/// See more: +/// * [MediaPipe's Category documentation](https://developers.google.com/mediapipe/api/solutions/java/com/google/mediapipe/tasks/components/containers/Category) +class Category { + /// Generative constructor that creates a [Category] instance. + const Category({ + required this.index, + required this.score, + required this.categoryName, + required this.displayName, + }); + + /// The index of the label in the corresponding label file. + final int index; + + /// The probability score of this label category. + final double score; + + /// The label of this category object. + final String? categoryName; + + /// The display name of the label, which may be translated for different locales. + final String? displayName; + + /// Accepts a pointer to a list of structs, and a count representing the length + /// of the list, and returns a list of pure-Dart [Category] instances. + static List fromStructs( + Pointer structs, + int count, + ) { + final categories = []; + for (int i = 0; i < count; i++) { + categories.add(fromStruct(structs[i])); + } + return categories; + } + + /// Accepts a pointer to a single struct and returns a pure-Dart [Category] instance. + static Category fromStruct(bindings.Category struct) { + return Category( + index: struct.index, + score: struct.score, + categoryName: toDartString(struct.category_name), + displayName: toDartString(struct.display_name), + ); + } + + /// Releases all C memory associated with a list of [bindings.Category] pointers. + /// This method is important to call after calling [Category.fromStructs] to + /// convert that C memory into pure-Dart objects. + static void freeStructs(Pointer structs, int count) { + int index = 0; + while (index < count) { + bindings.Category obj = structs[index]; + calloc.free(obj.category_name); + calloc.free(obj.display_name); + index++; + } + calloc.free(structs); + } + + @override + String toString() => 'Category(index=$index, score=$score, ' + 'categoryName=$categoryName, displayName=$displayName)'; +} + +/// Dart representation of MediaPipe's "Classifications" concept. +/// +/// Represents the list of classification for a given classifier head. +/// Typically used as a result for classification tasks. +/// +/// See also: +/// * [MediaPipe's Classifications documentation](https://developers.google.com/mediapipe/api/solutions/java/com/google/mediapipe/tasks/components/containers/Classifications) +class Classifications { + /// Generative constructor that creates a [Classifications] instance. + const Classifications({ + required this.categories, + required this.headIndex, + required this.headName, + }); + + /// A list of [Category] objects which contain the actual classification + /// information, including human-readable labels and probability scores. + final List categories; + + /// The index of the classifier head these entries refer to. + final int headIndex; + + /// The optional name of the classifier head, which is the corresponding + /// tensor metadata name. + final String? headName; + + /// Accepts a pointer to a list of structs, and a count representing the length + /// of the list, and returns a list of pure-Dart [Classifications] instances. + static List fromStructs( + Pointer structs, + int count, + ) { + final classifications = []; + for (int i = 0; i < count; i++) { + classifications.add(fromStruct(structs[i])); + } + return classifications; + } + + /// Accepts a pointer to a single struct and returns a pure-Dart [Classifications] + /// instance. + static Classifications fromStruct(bindings.Classifications struct) { + return Classifications( + categories: Category.fromStructs( + struct.categories, + struct.categories_count, + ), + headIndex: struct.head_index, + headName: toDartString(struct.head_name), + ); + } + + /// Releases all C memory associated with a list of [bindings.Classifications] + /// pointers. This method is important to call after calling [Classifications.fromStructs] + /// to convert that C memory into pure-Dart objects. + static void freeStructs( + Pointer structs, + int count, + ) { + int index = 0; + while (index < count) { + bindings.Classifications obj = structs[index]; + Category.freeStructs(obj.categories, obj.categories_count); + calloc.free(obj.head_name); + index++; + } + calloc.free(structs); + } + + /// Convenience getter for the first [Category] out of the [categories] list. + Category? get firstCategory => + categories.isNotEmpty ? categories.first : null; + + @override + String toString() { + final categoryStrings = categories.map((cat) => cat.toString()).join(', '); + return 'Classification(categories=[$categoryStrings], ' + 'headIndex=$headIndex, headName=$headName)'; + } +} diff --git a/packages/mediapipe-core/lib/src/ffi_utils.dart b/packages/mediapipe-core/lib/src/ffi_utils.dart new file mode 100644 index 0000000..a5411c9 --- /dev/null +++ b/packages/mediapipe-core/lib/src/ffi_utils.dart @@ -0,0 +1,78 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:ffi'; +import 'dart:typed_data'; +import 'package:ffi/ffi.dart'; + +/// Converts a list of Dart strings into their C memory equivalent. +/// +/// See also: +/// * [prepareString] +Pointer> prepareListOfStrings(List values) { + final ptrArray = calloc>(values.length); + for (var i = 0; i < values.length; i++) { + ptrArray[i] = prepareString(values[i]); + } + return ptrArray; +} + +/// Converts a single Dart string into its C memory equivalent. +/// +/// See also: +/// * [prepareListOfStrings] +Pointer prepareString(String val) => val.toNativeUtf8().cast(); + +/// Converts the C memory representation of a string into a Dart [String]. If the +/// supplied pointer is a null pointer, this function returns `null`. +/// +/// See also: +/// * [toDartStrings] +String? toDartString(Pointer val) { + if (val.address == 0) return null; + return val.cast().toDartString(); +} + +/// Converts a list of C memory representations of strings into a list of Dart +/// [String]s. +/// +/// See also: +/// * [toDartString] +List toDartStrings(Pointer> val, int length) { + final dartStrings = []; + int counter = 0; + while (counter < length) { + dartStrings.add(toDartString(val[counter])); + counter++; + } + return dartStrings; +} + +/// Converts Dart's representation for binary data, a [Uint8List], into its C +/// memory representation. +Pointer prepareUint8List(Uint8List ints) { + final Pointer ptr = calloc(ints.length); + ptr.asTypedList(ints.length).setAll(0, ints); + return ptr.cast(); +} + +/// Converts a pointer to binary data in C memory to Dart's representation for +/// binary data, a [Uint8List]. +Uint8List toUint8List(Pointer val, {int? length}) { + final codeUnits = val.cast(); + if (length != null) { + RangeError.checkNotNegative(length, 'length'); + } else { + length = _length(codeUnits); + } + return codeUnits.asTypedList(length); +} + +int _length(Pointer codeUnits) { + var length = 0; + while (codeUnits[length] != 0) { + length++; + } + return length; +} diff --git a/packages/mediapipe-core/lib/src/task_options.dart b/packages/mediapipe-core/lib/src/task_options.dart new file mode 100644 index 0000000..c12ff61 --- /dev/null +++ b/packages/mediapipe-core/lib/src/task_options.dart @@ -0,0 +1,175 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:ffi'; +import 'dart:typed_data'; +import 'package:equatable/equatable.dart'; +import 'package:ffi/ffi.dart'; +import 'ffi_utils.dart'; +import 'third_party/mediapipe/generated/mediapipe_common_bindings.dart' + as bindings; + +/// Dart representation of MediaPipe's "BaseOptions" concept. +/// +/// Used to configure various classifiers by specifying the model they will use +/// for computation. +/// +/// See also: +/// * [MediaPipe's BaseOptions documentation](https://developers.google.com/mediapipe/api/solutions/java/com/google/mediapipe/tasks/core/BaseOptions) +/// * [ClassifierOptions], which is often used in conjunction to specify a +/// classifier's desired behavior. +class BaseOptions extends Equatable { + /// Generative constructor that creates a [BaseOptions] instance. + const BaseOptions({this.modelAssetBuffer, this.modelAssetPath}) + : assert( + !(modelAssetBuffer == null && modelAssetPath == null), + 'You must supply either `modelAssetBuffer` or `modelAssetPath`', + ), + assert( + !(modelAssetBuffer != null && modelAssetPath != null), + 'You must only supply one of `modelAssetBuffer` and `modelAssetPath`', + ); + + /// The model asset file contents as bytes; + final Uint8List? modelAssetBuffer; + + /// Path to the model asset file. + final String? modelAssetPath; + + /// Converts this pure-Dart representation into C-memory suitable for the + /// MediaPipe SDK to instantiate various classifiers. + Pointer toStruct() { + final struct = calloc(); + + if (modelAssetPath != null) { + struct.ref.model_asset_path = prepareString(modelAssetPath!); + } + if (modelAssetBuffer != null) { + struct.ref.model_asset_buffer = prepareUint8List(modelAssetBuffer!); + } + return struct; + } + + @override + List get props => [modelAssetBuffer, modelAssetPath]; + + /// Releases all C memory held by this [bindings.BaseOptions] struct. + static void freeStruct(bindings.BaseOptions struct) { + if (struct.model_asset_buffer.address != 0) { + calloc.free(struct.model_asset_buffer); + } + if (struct.model_asset_path.address != 0) { + calloc.free(struct.model_asset_path); + } + } +} + +/// Dart representation of MediaPipe's "ClassifierOptions" concept. +/// +/// Classifier options shared across MediaPipe classification tasks. +/// +/// See also: +/// * [MediaPipe's ClassifierOptions documentation](https://developers.google.com/mediapipe/api/solutions/java/com/google/mediapipe/tasks/components/processors/ClassifierOptions) +/// * [BaseOptions], which is often used in conjunction to specify a +/// classifier's desired behavior. +class ClassifierOptions extends Equatable { + /// Generative constructor that creates a [ClassifierOptions] instance. + const ClassifierOptions({ + this.displayNamesLocale, + this.maxResults, + this.scoreThreshold, + this.categoryAllowlist, + this.categoryDenylist, + }); + + /// The locale to use for display names specified through the TFLite Model + /// Metadata. + final String? displayNamesLocale; + + /// The maximum number of top-scored classification results to return. + final int? maxResults; + + /// If set, establishes a minimum `score` and leads to the rejection of any + /// categories with lower `score` values. + final double? scoreThreshold; + + /// Allowlist of category names. + /// + /// If non-empty, classification results whose category name is not in + /// this set will be discarded. Duplicate or unknown category names + /// are ignored. Mutually exclusive with `categoryDenylist`. + final List? categoryAllowlist; + + /// Denylist of category names. + /// + /// If non-empty, classification results whose category name is in this set + /// will be discarded. Duplicate or unknown category names are ignored. + /// Mutually exclusive with `categoryAllowList`. + final List? categoryDenylist; + + /// Converts this pure-Dart representation into C-memory suitable for the + /// MediaPipe SDK to instantiate various classifiers. + Pointer toStruct() { + final struct = calloc(); + _setDisplayNamesLocale(struct.ref); + _setMaxResults(struct.ref); + _setScoreThreshold(struct.ref); + _setAllowlist(struct.ref); + _setDenylist(struct.ref); + return struct; + } + + void _setDisplayNamesLocale(bindings.ClassifierOptions struct) { + if (displayNamesLocale != null) { + struct.display_names_locale = prepareString(displayNamesLocale!); + } + } + + void _setMaxResults(bindings.ClassifierOptions struct) { + // This value must not be zero, and -1 implies no limit. + struct.max_results = maxResults ?? -1; + } + + void _setScoreThreshold(bindings.ClassifierOptions struct) { + if (scoreThreshold != null) { + struct.score_threshold = scoreThreshold!; + } + } + + void _setAllowlist(bindings.ClassifierOptions struct) { + if (categoryAllowlist != null) { + struct.category_allowlist = prepareListOfStrings(categoryAllowlist!); + struct.category_allowlist_count = categoryAllowlist!.length; + } + } + + void _setDenylist(bindings.ClassifierOptions struct) { + if (categoryDenylist != null) { + struct.category_denylist = prepareListOfStrings(categoryDenylist!); + struct.category_denylist_count = categoryDenylist!.length; + } + } + + /// Releases all C memory held by this [bindings.ClassifierOptions] struct. + static void freeStruct(bindings.ClassifierOptions struct) { + if (struct.display_names_locale.address != 0) { + calloc.free(struct.display_names_locale); + } + if (struct.category_allowlist.address != 0) { + calloc.free(struct.category_allowlist); + } + if (struct.category_denylist.address != 0) { + calloc.free(struct.category_denylist); + } + } + + @override + List get props => [ + displayNamesLocale, + maxResults, + scoreThreshold, + ...(categoryAllowlist ?? []), + ...(categoryDenylist ?? []), + ]; +} diff --git a/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart b/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart new file mode 100644 index 0000000..13bf3d2 --- /dev/null +++ b/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart @@ -0,0 +1,347 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// AUTO GENERATED FILE, DO NOT EDIT. +// +// Generated by `package:ffigen`. +// ignore_for_file: type=lint +import 'dart:ffi' as ffi; + +final class BaseOptions extends ffi.Struct { + external ffi.Pointer model_asset_buffer; + + external ffi.Pointer model_asset_path; +} + +final class __mbstate_t extends ffi.Union { + @ffi.Array.multi([128]) + external ffi.Array __mbstate8; + + @ffi.LongLong() + external int _mbstateL; +} + +final class __darwin_pthread_handler_rec extends ffi.Struct { + external ffi + .Pointer)>> + __routine; + + external ffi.Pointer __arg; + + external ffi.Pointer<__darwin_pthread_handler_rec> __next; +} + +final class _opaque_pthread_attr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([56]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_cond_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([40]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_condattr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([8]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_mutex_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([56]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_mutexattr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([8]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_once_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([8]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_rwlock_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([192]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_rwlockattr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([16]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + external ffi.Pointer<__darwin_pthread_handler_rec> __cleanup_stack; + + @ffi.Array.multi([8176]) + external ffi.Array __opaque; +} + +final class ClassifierOptions extends ffi.Struct { + external ffi.Pointer display_names_locale; + + @ffi.Int() + external int max_results; + + @ffi.Float() + external double score_threshold; + + external ffi.Pointer> category_allowlist; + + @ffi.Uint32() + external int category_allowlist_count; + + external ffi.Pointer> category_denylist; + + @ffi.Uint32() + external int category_denylist_count; +} + +final class Category extends ffi.Struct { + @ffi.Int() + external int index; + + @ffi.Float() + external double score; + + external ffi.Pointer category_name; + + external ffi.Pointer display_name; +} + +final class Classifications extends ffi.Struct { + external ffi.Pointer categories; + + @ffi.Uint32() + external int categories_count; + + @ffi.Int() + external int head_index; + + external ffi.Pointer head_name; +} + +final class ClassificationResult extends ffi.Struct { + external ffi.Pointer classifications; + + @ffi.Uint32() + external int classifications_count; + + @ffi.Int64() + external int timestamp_ms; + + @ffi.Bool() + external bool has_timestamp_ms; +} + +const int __WORDSIZE = 64; + +const int __DARWIN_ONLY_64_BIT_INO_T = 1; + +const int __DARWIN_ONLY_UNIX_CONFORMANCE = 1; + +const int __DARWIN_ONLY_VERS_1050 = 1; + +const int __DARWIN_UNIX03 = 1; + +const int __DARWIN_64_BIT_INO_T = 1; + +const int __DARWIN_VERS_1050 = 1; + +const int __DARWIN_NON_CANCELABLE = 0; + +const String __DARWIN_SUF_EXTSN = '\$DARWIN_EXTSN'; + +const int __DARWIN_C_ANSI = 4096; + +const int __DARWIN_C_FULL = 900000; + +const int __DARWIN_C_LEVEL = 900000; + +const int __STDC_WANT_LIB_EXT1__ = 1; + +const int __DARWIN_NO_LONG_LONG = 0; + +const int _DARWIN_FEATURE_64_BIT_INODE = 1; + +const int _DARWIN_FEATURE_ONLY_64_BIT_INODE = 1; + +const int _DARWIN_FEATURE_ONLY_VERS_1050 = 1; + +const int _DARWIN_FEATURE_ONLY_UNIX_CONFORMANCE = 1; + +const int _DARWIN_FEATURE_UNIX_CONFORMANCE = 3; + +const int __has_ptrcheck = 0; + +const int __DARWIN_NULL = 0; + +const int __PTHREAD_SIZE__ = 8176; + +const int __PTHREAD_ATTR_SIZE__ = 56; + +const int __PTHREAD_MUTEXATTR_SIZE__ = 8; + +const int __PTHREAD_MUTEX_SIZE__ = 56; + +const int __PTHREAD_CONDATTR_SIZE__ = 8; + +const int __PTHREAD_COND_SIZE__ = 40; + +const int __PTHREAD_ONCE_SIZE__ = 8; + +const int __PTHREAD_RWLOCK_SIZE__ = 192; + +const int __PTHREAD_RWLOCKATTR_SIZE__ = 16; + +const int USER_ADDR_NULL = 0; + +const int INT8_MAX = 127; + +const int INT16_MAX = 32767; + +const int INT32_MAX = 2147483647; + +const int INT64_MAX = 9223372036854775807; + +const int INT8_MIN = -128; + +const int INT16_MIN = -32768; + +const int INT32_MIN = -2147483648; + +const int INT64_MIN = -9223372036854775808; + +const int UINT8_MAX = 255; + +const int UINT16_MAX = 65535; + +const int UINT32_MAX = 4294967295; + +const int UINT64_MAX = -1; + +const int INT_LEAST8_MIN = -128; + +const int INT_LEAST16_MIN = -32768; + +const int INT_LEAST32_MIN = -2147483648; + +const int INT_LEAST64_MIN = -9223372036854775808; + +const int INT_LEAST8_MAX = 127; + +const int INT_LEAST16_MAX = 32767; + +const int INT_LEAST32_MAX = 2147483647; + +const int INT_LEAST64_MAX = 9223372036854775807; + +const int UINT_LEAST8_MAX = 255; + +const int UINT_LEAST16_MAX = 65535; + +const int UINT_LEAST32_MAX = 4294967295; + +const int UINT_LEAST64_MAX = -1; + +const int INT_FAST8_MIN = -128; + +const int INT_FAST16_MIN = -32768; + +const int INT_FAST32_MIN = -2147483648; + +const int INT_FAST64_MIN = -9223372036854775808; + +const int INT_FAST8_MAX = 127; + +const int INT_FAST16_MAX = 32767; + +const int INT_FAST32_MAX = 2147483647; + +const int INT_FAST64_MAX = 9223372036854775807; + +const int UINT_FAST8_MAX = 255; + +const int UINT_FAST16_MAX = 65535; + +const int UINT_FAST32_MAX = 4294967295; + +const int UINT_FAST64_MAX = -1; + +const int INTPTR_MAX = 9223372036854775807; + +const int INTPTR_MIN = -9223372036854775808; + +const int UINTPTR_MAX = -1; + +const int INTMAX_MAX = 9223372036854775807; + +const int UINTMAX_MAX = -1; + +const int INTMAX_MIN = -9223372036854775808; + +const int PTRDIFF_MIN = -9223372036854775808; + +const int PTRDIFF_MAX = 9223372036854775807; + +const int SIZE_MAX = -1; + +const int RSIZE_MAX = 9223372036854775807; + +const int WCHAR_MAX = 2147483647; + +const int WCHAR_MIN = -2147483648; + +const int WINT_MIN = -2147483648; + +const int WINT_MAX = 2147483647; + +const int SIG_ATOMIC_MIN = -2147483648; + +const int SIG_ATOMIC_MAX = 2147483647; + +const int __bool_true_false_are_defined = 1; + +const int true1 = 1; + +const int false1 = 0; diff --git a/packages/mediapipe-core/pubspec.yaml b/packages/mediapipe-core/pubspec.yaml new file mode 100644 index 0000000..501b789 --- /dev/null +++ b/packages/mediapipe-core/pubspec.yaml @@ -0,0 +1,20 @@ +name: mediapipe_core +description: Shared logic and utilities required by other MediaPipe Task packages. +version: 0.0.1-wip +publish_to: none + +repository: https://github.com/google/flutter-mediapipe +issue_tracker: https://github.com/google/flutter-mediapipe/issues + +environment: + sdk: ^3.1.0-333.0.dev + +# Add regular dependencies here. +dependencies: + equatable: ^2.0.5 + ffi: ^2.1.0 + +dev_dependencies: + ffigen: ^9.0.1 + lints: ^2.0.0 + test: ^1.21.0 diff --git a/packages/mediapipe-core/test/task_options_test.dart b/packages/mediapipe-core/test/task_options_test.dart new file mode 100644 index 0000000..143a9ff --- /dev/null +++ b/packages/mediapipe-core/test/task_options_test.dart @@ -0,0 +1,110 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:ffi'; +import 'dart:typed_data'; + +import 'package:test/test.dart'; +import 'package:mediapipe_core/mediapipe_core.dart'; + +void main() { + group('BaseOptions constructor should', () { + test('enforce exactly one of modelPath and modelBuffer', () { + expect( + () => BaseOptions( + modelAssetPath: 'abc', + modelAssetBuffer: Uint8List.fromList([1, 2, 3]), + ), + throwsA(TypeMatcher()), + ); + + expect(BaseOptions.new, throwsA(TypeMatcher())); + }); + }); + + group('BaseOptions.toStruct/fromStruct should', () { + test('allocate memory in C for a modelAssetPath', () { + final options = BaseOptions(modelAssetPath: 'abc'); + final struct = options.toStruct(); + expect(toDartString(struct.ref.model_asset_path), 'abc'); + expectNullPtr(struct.ref.model_asset_buffer); + }); + + test('allocate memory in C for a modelAssetBuffer', () { + final options = BaseOptions( + modelAssetBuffer: Uint8List.fromList([1, 2, 3]), + ); + final struct = options.toStruct(); + expect( + toUint8List(struct.ref.model_asset_buffer), + Uint8List.fromList([1, 2, 3]), + ); + expectNullPtr(struct.ref.model_asset_path); + }); + + test('allocate memory in C for a modelAssetBuffer containing 0', () { + final options = BaseOptions( + modelAssetBuffer: Uint8List.fromList([1, 2, 0, 3]), + ); + final struct = options.toStruct(); + expect( + toUint8List(struct.ref.model_asset_buffer), + Uint8List.fromList([1, 2]), + ); + expectNullPtr(struct.ref.model_asset_path); + }); + }); + + group('ClassOptions should', () { + test('allocate memory for empty fields', () { + final options = ClassifierOptions(); + + final struct = options.toStruct(); + + expect(struct.ref.max_results, -1); + expect(struct.ref.score_threshold, 0.0); + expectNullPtr(struct.ref.category_allowlist); + expect(struct.ref.category_allowlist_count, 0); + expectNullPtr(struct.ref.category_denylist); + expect(struct.ref.category_denylist_count, 0); + expectNullPtr(struct.ref.display_names_locale); + }); + + test('allocate memory for full fields', () { + final options = ClassifierOptions( + displayNamesLocale: 'en', + maxResults: 5, + scoreThreshold: 0.9, + categoryAllowlist: ['good', 'great', 'best'], + categoryDenylist: ['bad', 'terrible', 'worst', 'honestly come on'], + ); + + final struct = options.toStruct(); + + expect(toDartString(struct.ref.display_names_locale), 'en'); + expect(struct.ref.max_results, 5); + expect(struct.ref.score_threshold, greaterThan(0.8999)); + expect(struct.ref.score_threshold, lessThan(0.90001)); + expect( + toDartStrings( + struct.ref.category_allowlist, + struct.ref.category_allowlist_count, + ), + ['good', 'great', 'best'], + ); + expect(struct.ref.category_allowlist_count, 3); + + expect( + toDartStrings( + struct.ref.category_denylist, + struct.ref.category_denylist_count, + ), + ['bad', 'terrible', 'worst', 'honestly come on'], + ); + expect(struct.ref.category_denylist_count, 4); + }); + }); +} + +void expectNullPtr(Pointer ptr) => expect(ptr.address, equals(0)); diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/category.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/category.h new file mode 100644 index 0000000..b6eede4 --- /dev/null +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/category.h @@ -0,0 +1,50 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CATEGORY_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CATEGORY_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +// Defines a single classification result. +// +// The label maps packed into the TFLite Model Metadata [1] are used to populate +// the 'category_name' and 'display_name' fields. +// +// [1]: https://www.tensorflow.org/lite/convert/metadata +struct Category { + // The index of the category in the classification model output. + int index; + + // The score for this category, e.g. (but not necessarily) a probability in + // [0,1]. + float score; + + // The optional ID for the category, read from the label map packed in the + // TFLite Model Metadata if present. Not necessarily human-readable. + const char* category_name; + + // The optional human-readable name for the category, read from the label map + // packed in the TFLite Model Metadata if present. + const char* display_name; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CATEGORY_H_ diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h new file mode 100644 index 0000000..07ba44f --- /dev/null +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h @@ -0,0 +1,70 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CLASSIFICATION_RESULT_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CLASSIFICATION_RESULT_H_ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Defines classification results for a given classifier head. +struct Classifications { + // The array of predicted categories, usually sorted by descending scores, + // e.g. from high to low probability. + struct Category* categories; + + // The number of elements in the categories array. + uint32_t categories_count; + + // The index of the classifier head (i.e. output tensor) these categories + // refer to. This is useful for multi-head models. + int head_index; + + // The optional name of the classifier head, as provided in the TFLite Model + // Metadata [1] if present. This is useful for multi-head models. + // + // [1]: https://www.tensorflow.org/lite/convert/metadata + const char* head_name; +}; + +// Defines classification results of a model. +struct ClassificationResult { + // The classification results for each head of the model. + struct Classifications* classifications; + // The number of classifications in the classifications array. + uint32_t classifications_count; + + // The optional timestamp (in milliseconds) of the start of the chunk of data + // corresponding to these results. + // + // This is only used for classification on time series (e.g. audio + // classification). In these use cases, the amount of data to process might + // exceed the maximum size that the model can process: to solve this, the + // input data is split into multiple chunks starting at different timestamps. + int64_t timestamp_ms; + + // Specifies whether the timestamp contains a valid value. + bool has_timestamp_ms; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CLASSIFICATION_RESULT_H_ diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h new file mode 100644 index 0000000..1c153bd --- /dev/null +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h @@ -0,0 +1,61 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_PROCESSORS_CLASSIFIER_OPTIONS_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_PROCESSORS_CLASSIFIER_OPTIONS_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Classifier options for MediaPipe C classification Tasks. +struct ClassifierOptions { + // The locale to use for display names specified through the TFLite Model + // Metadata, if any. Defaults to English. + char* display_names_locale; + + // The maximum number of top-scored classification results to return. If < 0, + // all available results will be returned. If 0, an invalid argument error is + // returned. + int max_results; + + // Score threshold to override the one provided in the model metadata (if + // any). Results below this value are rejected. + float score_threshold; + + // The allowlist of category names. If non-empty, detection results whose + // category name is not in this set will be filtered out. Duplicate or unknown + // category names are ignored. Mutually exclusive with category_denylist. + char** category_allowlist; + + // The number of elements in the category allowlist. + uint32_t category_allowlist_count; + + // The denylist of category names. If non-empty, detection results whose + // category name is in this set will be filtered out. Duplicate or unknown + // category names are ignored. Mutually exclusive with category_allowlist. + + char** category_denylist; + // The number of elements in the category denylist. + uint32_t category_denylist_count; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_PROCESSORS_CLASSIFIER_OPTIONS_H_ diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h new file mode 100644 index 0000000..d23b688 --- /dev/null +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h @@ -0,0 +1,36 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_CORE_BASE_OPTIONS_H_ +#define MEDIAPIPE_TASKS_C_CORE_BASE_OPTIONS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +// Base options for MediaPipe C Tasks. +struct BaseOptions { + // The model asset file contents as a string. + char* model_asset_buffer; + + // The path to the model asset to open and mmap in memory. + char* model_asset_path; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_CORE_BASE_OPTIONS_H_ diff --git a/tool/ci_script_shared.sh b/tool/ci_script_shared.sh new file mode 100644 index 0000000..183f71a --- /dev/null +++ b/tool/ci_script_shared.sh @@ -0,0 +1,28 @@ +function ci_package () { + local channel="$1" + + shift + local arr=("$@") + for PACKAGE_NAME in "${arr[@]}" + do + echo "== Testing '${PACKAGE_NAME}' on Flutter's $channel channel ==" + pushd "packages/${PACKAGE_NAME}" + + # Grab packages. + flutter pub get + + # Run the analyzer to find any static analysis issues. + dart analyze --fatal-infos + + # Run the formatter on all the dart files to make sure everything's linted. + dart format --output none --set-exit-if-changed . + + # Run the actual tests. + if [ -d "test" ] + then + flutter test + fi + + popd + done +} diff --git a/tool/mediapipe_ci_script_beta.sh b/tool/mediapipe_ci_script_beta.sh new file mode 100755 index 0000000..5f914f4 --- /dev/null +++ b/tool/mediapipe_ci_script_beta.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +DIR="${BASH_SOURCE%/*}" +source "$DIR/ci_script_shared.sh" + +flutter doctor -v + +declare -ar PACKAGE_NAMES=( + "mediapipe-core" +) + +ci_package "beta" "${PACKAGE_NAMES[@]}" + +echo "-- Success --" diff --git a/tool/mediapipe_ci_script_master.sh b/tool/mediapipe_ci_script_master.sh new file mode 100755 index 0000000..48c5d65 --- /dev/null +++ b/tool/mediapipe_ci_script_master.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +DIR="${BASH_SOURCE%/*}" +source "$DIR/ci_script_shared.sh" + +flutter doctor -v + +declare -ar PACKAGE_NAMES=( + "mediapipe-core" +) + +ci_package "master" "${PACKAGE_NAMES[@]}" + +echo "-- Success --" diff --git a/tool/mediapipe_ci_script_stable.sh b/tool/mediapipe_ci_script_stable.sh new file mode 100755 index 0000000..153dda2 --- /dev/null +++ b/tool/mediapipe_ci_script_stable.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +DIR="${BASH_SOURCE%/*}" +source "$DIR/ci_script_shared.sh" + +flutter doctor -v + +declare -ar PACKAGE_NAMES=( + "mediapipe-core" +) + +ci_package "stable" "${PACKAGE_NAMES[@]}" + +echo "-- Success --"