Merge branch 'main' of https://github.com/microsoft/onnxruntime into …

…gemm8

cmake/onnxruntime_providers.cmake

@@ -1799,7 +1799,7 @@ if (onnxruntime_USE_XNNPACK)
   source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_providers_xnnpack_cc_srcs})
   onnxruntime_add_static_library(onnxruntime_providers_xnnpack ${onnxruntime_providers_xnnpack_cc_srcs})
   onnxruntime_add_include_to_target(onnxruntime_providers_xnnpack
-    onnxruntime_common onnxruntime_framework onnx onnx_proto ${PROTOBUF_LIB} XNNPACK pthreadpool Boost::mp11 safeint_interface
+    onnxruntime_common onnxruntime_framework onnx onnx_proto ${PROTOBUF_LIB} XNNPACK pthreadpool flatbuffers::flatbuffers Boost::mp11 safeint_interface
   )
 
   add_dependencies(onnxruntime_providers_xnnpack onnx ${onnxruntime_EXTERNAL_DEPENDENCIES})

cmake/onnxruntime_webassembly.cmake

@@ -277,19 +277,29 @@ else()
       "SHELL:-s EXPORT_NAME=ortWasmThreaded"
       "SHELL:-s DEFAULT_PTHREAD_STACK_SIZE=131072"
     )
-    if (onnxruntime_ENABLE_WEBASSEMBLY_SIMD)
-      set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm-simd-threaded")
-    else()
-      set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm-threaded")
-    endif()
   else()
     target_link_options(onnxruntime_webassembly PRIVATE
       "SHELL:-s EXPORT_NAME=ortWasm"
     )
-    if (onnxruntime_ENABLE_WEBASSEMBLY_SIMD)
-      set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm-simd")
-    else()
-      set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME "ort-wasm")
-    endif()
   endif()
+
+  set(target_name ort)
+
+  if (onnxruntime_ENABLE_TRAINING_APIS)
+    list(APPEND target_name "training")
+  endif()
+
+  list(APPEND target_name "wasm")
+
+  if (onnxruntime_ENABLE_WEBASSEMBLY_SIMD)
+    list(APPEND target_name "simd")
+  endif()
+
+  if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS)
+    list(APPEND target_name "threaded")
+  endif()
+
+  list(JOIN target_name "-" target_name)
+
+  set_target_properties(onnxruntime_webassembly PROPERTIES OUTPUT_NAME ${target_name})
 endif()

docs/OperatorKernels.md

@@ -759,6 +759,7 @@ Do not modify directly.*
 |Shrink|*in* input:**T**<br> *out* output:**T**|9+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)|
 |Sigmoid|*in* X:**T**<br> *out* Y:**T**|13+|**T** = tensor(bfloat16), tensor(double), tensor(float), tensor(float16)|
 |||[6, 12]|**T** = tensor(double), tensor(float), tensor(float16)|
+|Sign|*in* input:**T**<br> *out* output:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)|
 |SimplifiedLayerNormalization|*in* X:**T**<br> *in* scale:**V**<br> *out* Y:**V**<br> *out* inv_std_var:**U**|1+|**T** = tensor(double), tensor(float), tensor(float16)<br/> **U** = tensor(double), tensor(float)<br/> **V** = tensor(double), tensor(float), tensor(float16)|
 |Sin|*in* input:**T**<br> *out* output:**T**|7+|**T** = tensor(double), tensor(float), tensor(float16)|
 |Size|*in* data:**T**<br> *out* size:**T1**|13+|**T** = tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(string), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)<br/> **T1** = tensor(int64)|

include/onnxruntime/core/graph/graph.h

@@ -20,6 +20,8 @@
 #pragma warning(pop)
 #endif
 
+#include "flatbuffers/flatbuffers.h"
+
 #include "core/common/gsl.h"
 
 #include "core/common/common.h"
@@ -43,12 +45,6 @@
 #include "core/graph/node_arg.h"
 #include "core/graph/ort_format_load_options.h"
 
-namespace flatbuffers {
-class FlatBufferBuilder;
-template <typename T>
-struct Offset;
-}  // namespace flatbuffers
-
 namespace onnxruntime {
 class Graph;
 struct IndexedSubGraph;

js/common/lib/env.ts

@@ -61,6 +61,10 @@ export declare namespace Env {
      * @defaultValue `'webgl2'`
      */
     contextId?: 'webgl'|'webgl2';
+    /**
+     * Get the WebGL rendering context.
+     */
+    readonly context: WebGLRenderingContext;
     /**
      * Set or get the maximum batch size for matmul. 0 means to disable batching.
      *
@@ -88,7 +92,19 @@ export declare namespace Env {
   }
 
   export interface WebGpuFlags {
+    /**
+     * Set or get the profiling mode.
+     */
     profilingMode?: 'off'|'default';
+    /**
+     * Get the device for WebGPU.
+     *
+     * When use with TypeScript, the type of this property is `GPUDevice` defined in "@webgpu/types".
+     * Use `const device = env.webgpu.device as GPUDevice;` in TypeScript to access this property with correct type.
+     *
+     * see comments on {@link GpuBufferType} for more details about why not use types defined in "@webgpu/types".
+     */
+    readonly device: unknown;
   }
 }
 
@@ -110,27 +126,27 @@ export interface Env {
    * Get version of the current package.
    */
   readonly versions: {
-    common: string;
-    web?: string;
-    node?: string;
+    readonly common: string;
+    readonly web?: string;
+    readonly node?: string;
     // eslint-disable-next-line @typescript-eslint/naming-convention
-    'react-native'?: string;
+    readonly 'react-native'?: string;
   };
 
   /**
    * Represent a set of flags for WebAssembly
    */
-  wasm: Env.WebAssemblyFlags;
+  readonly wasm: Env.WebAssemblyFlags;
 
   /**
    * Represent a set of flags for WebGL
    */
-  webgl: Env.WebGLFlags;
+  readonly webgl: Env.WebGLFlags;
 
   /**
    * Represent a set of flags for WebGPU
    */
-  webgpu: Env.WebGpuFlags;
+  readonly webgpu: Env.WebGpuFlags;
 
   [name: string]: unknown;
 }

js/common/lib/inference-session.ts

@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 import {InferenceSession as InferenceSessionImpl} from './inference-session-impl.js';
-import {OnnxValue} from './onnx-value.js';
+import {OnnxValue, OnnxValueDataLocation} from './onnx-value.js';
 
 /* eslint-disable @typescript-eslint/no-redeclare */
 
@@ -138,6 +138,14 @@ export declare namespace InferenceSession {
      */
     logVerbosityLevel?: number;
 
+    /**
+     * Specify string as a preferred data location for all outputs, or an object that use output names as keys and a
+     * preferred data location as corresponding values.
+     *
+     * This setting is available only in ONNXRuntime Web for WebGL and WebGPU EP.
+     */
+    preferredOutputLocation?: OnnxValueDataLocation|{readonly [outputName: string]: OnnxValueDataLocation};
+
     /**
      * Store configurations for a session. See
      * https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/session/

js/common/lib/onnx-value.ts

@@ -11,3 +11,8 @@ type NonTensorType = never;
  * NOTE: currently not support non-tensor
  */
 export type OnnxValue = Tensor|NonTensorType;
+
+/**
+ * Type OnnxValueDataLocation represents the location of the data of an OnnxValue.
+ */
+export type OnnxValueDataLocation = Tensor.DataLocation;

js/common/lib/tensor-factory-impl.ts

@@ -1,8 +1,9 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-import {OptionsDimensions, OptionsFormat, OptionsNormalizationParameters, OptionsTensorFormat, OptionsTensorLayout, TensorFromImageBitmapOptions, TensorFromImageDataOptions, TensorFromImageElementOptions, TensorFromUrlOptions} from './tensor-factory.js';
-import {Tensor, TypedTensor} from './tensor.js';
+import {GpuBufferDataTypes, OptionsDimensions, OptionsFormat, OptionsNormalizationParameters, OptionsTensorFormat, OptionsTensorLayout, TensorFromGpuBufferOptions, TensorFromImageBitmapOptions, TensorFromImageDataOptions, TensorFromImageElementOptions, TensorFromTextureOptions, TensorFromUrlOptions, TextureDataTypes} from './tensor-factory.js';
+import {Tensor} from './tensor-impl.js';
+import {Tensor as TensorInterface} from './tensor.js';
 
 interface BufferToTensorOptions extends OptionsDimensions, OptionsTensorLayout, OptionsNormalizationParameters,
                                         OptionsFormat, OptionsTensorFormat {}
@@ -14,95 +15,92 @@ interface BufferToTensorOptions extends OptionsDimensions, OptionsTensorLayout,
  * @param imageFormat - input image configuration - required configurations height, width, format
  * @param tensorFormat - output tensor configuration - Default is RGB format
  */
-export const bufferToTensor =
-    (buffer: Uint8ClampedArray|undefined, options: BufferToTensorOptions): TypedTensor<'float32'>|
-    TypedTensor<'uint8'> => {
-      if (buffer === undefined) {
-        throw new Error('Image buffer must be defined');
-      }
-      if (options.height === undefined || options.width === undefined) {
-        throw new Error('Image height and width must be defined');
-      }
-      if (options.tensorLayout === 'NHWC') {
-        throw new Error('NHWC Tensor layout is not supported yet');
-      }
+export const bufferToTensor = (buffer: Uint8ClampedArray|undefined, options: BufferToTensorOptions): Tensor => {
+  if (buffer === undefined) {
+    throw new Error('Image buffer must be defined');
+  }
+  if (options.height === undefined || options.width === undefined) {
+    throw new Error('Image height and width must be defined');
+  }
+  if (options.tensorLayout === 'NHWC') {
+    throw new Error('NHWC Tensor layout is not supported yet');
+  }
 
-      const {height, width} = options;
+  const {height, width} = options;
 
-      const norm = options.norm ?? {mean: 255, bias: 0};
-      let normMean: [number, number, number, number];
-      let normBias: [number, number, number, number];
+  const norm = options.norm ?? {mean: 255, bias: 0};
+  let normMean: [number, number, number, number];
+  let normBias: [number, number, number, number];
 
-      if (typeof (norm.mean) === 'number') {
-        normMean = [norm.mean, norm.mean, norm.mean, norm.mean];
-      } else {
-        normMean = [norm.mean![0], norm.mean![1], norm.mean![2], norm.mean![3] ?? 255];
-      }
+  if (typeof (norm.mean) === 'number') {
+    normMean = [norm.mean, norm.mean, norm.mean, norm.mean];
+  } else {
+    normMean = [norm.mean![0], norm.mean![1], norm.mean![2], norm.mean![3] ?? 255];
+  }
 
-      if (typeof (norm.bias) === 'number') {
-        normBias = [norm.bias, norm.bias, norm.bias, norm.bias];
-      } else {
-        normBias = [norm.bias![0], norm.bias![1], norm.bias![2], norm.bias![3] ?? 0];
-      }
+  if (typeof (norm.bias) === 'number') {
+    normBias = [norm.bias, norm.bias, norm.bias, norm.bias];
+  } else {
+    normBias = [norm.bias![0], norm.bias![1], norm.bias![2], norm.bias![3] ?? 0];
+  }
 
-      const inputformat = options.format !== undefined ? options.format : 'RGBA';
-      // default value is RGBA since imagedata and HTMLImageElement uses it
-
-      const outputformat = options.tensorFormat !== undefined ?
-          (options.tensorFormat !== undefined ? options.tensorFormat : 'RGB') :
-          'RGB';
-      const stride = height * width;
-      const float32Data = outputformat === 'RGBA' ? new Float32Array(stride * 4) : new Float32Array(stride * 3);
-
-      // Default pointer assignments
-      let step = 4, rImagePointer = 0, gImagePointer = 1, bImagePointer = 2, aImagePointer = 3;
-      let rTensorPointer = 0, gTensorPointer = stride, bTensorPointer = stride * 2, aTensorPointer = -1;
-
-      // Updating the pointer assignments based on the input image format
-      if (inputformat === 'RGB') {
-        step = 3;
-        rImagePointer = 0;
-        gImagePointer = 1;
-        bImagePointer = 2;
-        aImagePointer = -1;
-      }
+  const inputformat = options.format !== undefined ? options.format : 'RGBA';
+  // default value is RGBA since imagedata and HTMLImageElement uses it
 
-      // Updating the pointer assignments based on the output tensor format
-      if (outputformat === 'RGBA') {
-        aTensorPointer = stride * 3;
-      } else if (outputformat === 'RBG') {
-        rTensorPointer = 0;
-        bTensorPointer = stride;
-        gTensorPointer = stride * 2;
-      } else if (outputformat === 'BGR') {
-        bTensorPointer = 0;
-        gTensorPointer = stride;
-        rTensorPointer = stride * 2;
-      }
+  const outputformat =
+      options.tensorFormat !== undefined ? (options.tensorFormat !== undefined ? options.tensorFormat : 'RGB') : 'RGB';
+  const stride = height * width;
+  const float32Data = outputformat === 'RGBA' ? new Float32Array(stride * 4) : new Float32Array(stride * 3);
 
-      for (let i = 0; i < stride;
-           i++, rImagePointer += step, bImagePointer += step, gImagePointer += step, aImagePointer += step) {
-        float32Data[rTensorPointer++] = (buffer[rImagePointer] + normBias[0]) / normMean[0];
-        float32Data[gTensorPointer++] = (buffer[gImagePointer] + normBias[1]) / normMean[1];
-        float32Data[bTensorPointer++] = (buffer[bImagePointer] + normBias[2]) / normMean[2];
-        if (aTensorPointer !== -1 && aImagePointer !== -1) {
-          float32Data[aTensorPointer++] = (buffer[aImagePointer] + normBias[3]) / normMean[3];
-        }
-      }
+  // Default pointer assignments
+  let step = 4, rImagePointer = 0, gImagePointer = 1, bImagePointer = 2, aImagePointer = 3;
+  let rTensorPointer = 0, gTensorPointer = stride, bTensorPointer = stride * 2, aTensorPointer = -1;
 
-      // Float32Array -> ort.Tensor
-      const outputTensor = outputformat === 'RGBA' ? new Tensor('float32', float32Data, [1, 4, height, width]) :
-                                                     new Tensor('float32', float32Data, [1, 3, height, width]);
-      return outputTensor;
-    };
+  // Updating the pointer assignments based on the input image format
+  if (inputformat === 'RGB') {
+    step = 3;
+    rImagePointer = 0;
+    gImagePointer = 1;
+    bImagePointer = 2;
+    aImagePointer = -1;
+  }
+
+  // Updating the pointer assignments based on the output tensor format
+  if (outputformat === 'RGBA') {
+    aTensorPointer = stride * 3;
+  } else if (outputformat === 'RBG') {
+    rTensorPointer = 0;
+    bTensorPointer = stride;
+    gTensorPointer = stride * 2;
+  } else if (outputformat === 'BGR') {
+    bTensorPointer = 0;
+    gTensorPointer = stride;
+    rTensorPointer = stride * 2;
+  }
+
+  for (let i = 0; i < stride;
+       i++, rImagePointer += step, bImagePointer += step, gImagePointer += step, aImagePointer += step) {
+    float32Data[rTensorPointer++] = (buffer[rImagePointer] + normBias[0]) / normMean[0];
+    float32Data[gTensorPointer++] = (buffer[gImagePointer] + normBias[1]) / normMean[1];
+    float32Data[bTensorPointer++] = (buffer[bImagePointer] + normBias[2]) / normMean[2];
+    if (aTensorPointer !== -1 && aImagePointer !== -1) {
+      float32Data[aTensorPointer++] = (buffer[aImagePointer] + normBias[3]) / normMean[3];
+    }
+  }
+
+  // Float32Array -> ort.Tensor
+  const outputTensor = outputformat === 'RGBA' ? new Tensor('float32', float32Data, [1, 4, height, width]) :
+                                                 new Tensor('float32', float32Data, [1, 3, height, width]);
+  return outputTensor;
+};
 
 /**
  * implementation of Tensor.fromImage().
  */
 export const tensorFromImage = async(
     image: ImageData|HTMLImageElement|ImageBitmap|string,
     options?: TensorFromImageDataOptions|TensorFromImageElementOptions|TensorFromImageBitmapOptions|
-    TensorFromUrlOptions): Promise<TypedTensor<'float32'>|TypedTensor<'uint8'>> => {
+    TensorFromUrlOptions): Promise<Tensor> => {
   // checking the type of image object
   const isHTMLImageEle = typeof (HTMLImageElement) !== 'undefined' && image instanceof HTMLImageElement;
   const isImageDataEle = typeof (ImageData) !== 'undefined' && image instanceof ImageData;
@@ -237,3 +235,30 @@ export const tensorFromImage = async(
     throw new Error('Input data provided is not supported - aborted tensor creation');
   }
 };
+
+/**
+ * implementation of Tensor.fromTexture().
+ */
+export const tensorFromTexture = <T extends TextureDataTypes>(
+    texture: TensorInterface.TextureType, options: TensorFromTextureOptions<T>): Tensor => {
+  const {width, height, download, dispose} = options;
+  // Always assume RGBAF32. TODO: support different texture format
+  const dims = [1, height, width, 4];
+  return new Tensor({location: 'texture', type: 'float32', texture, dims, download, dispose});
+};
+
+/**
+ * implementation of Tensor.fromGpuBuffer().
+ */
+export const tensorFromGpuBuffer = <T extends GpuBufferDataTypes>(
+    gpuBuffer: TensorInterface.GpuBufferType, options: TensorFromGpuBufferOptions<T>): Tensor => {
+  const {dataType, dims, download, dispose} = options;
+  return new Tensor({location: 'gpu-buffer', type: dataType ?? 'float32', gpuBuffer, dims, download, dispose});
+};
+
+/**
+ * implementation of Tensor.fromPinnedBuffer().
+ */
+export const tensorFromPinnedBuffer = <T extends Exclude<TensorInterface.Type, 'string'>>(
+    type: T, buffer: TensorInterface.DataTypeMap[T], dims?: readonly number[]): Tensor =>
+    new Tensor({location: 'cpu-pinned', type, data: buffer, dims: dims ?? [buffer.length]});