fix merge conflicts

cmake/onnxruntime_mlas.cmake

@@ -584,7 +584,7 @@ set_target_properties(onnxruntime_mlas PROPERTIES FOLDER "ONNXRuntime")
 if (WIN32)
   target_compile_options(onnxruntime_mlas PRIVATE "$<$<COMPILE_LANGUAGE:CXX>:/wd6385>" "$<$<COMPILE_LANGUAGE:CXX>:/wd4127>")
   if (onnxruntime_ENABLE_STATIC_ANALYSIS)
-    target_compile_options(onnxruntime_mlas PRIVATE  "$<$<COMPILE_LANGUAGE:CXX>:/analyze:stacksize 131072">)
+    target_compile_options(onnxruntime_mlas PRIVATE  "$<$<COMPILE_LANGUAGE:CXX>:/analyze:stacksize" 131072>)
   endif()
 endif()
 

cmake/onnxruntime_optimizer.cmake

@@ -42,12 +42,18 @@ if (onnxruntime_MINIMAL_BUILD)
       "${ONNXRUNTIME_ROOT}/core/optimizer/selectors_actions/selector_action_transformer_apply_contexts.h"
       "${ONNXRUNTIME_ROOT}/core/optimizer/selectors_actions/selector_action_transformer.cc"
       "${ONNXRUNTIME_ROOT}/core/optimizer/selectors_actions/selector_action_transformer.h"
-      "${ONNXRUNTIME_ROOT}/core/optimizer/transpose_optimizer/optimizer_api_impl.cc"
-      "${ONNXRUNTIME_ROOT}/core/optimizer/transpose_optimizer/optimizer_api.h"
-      "${ONNXRUNTIME_ROOT}/core/optimizer/transpose_optimizer/optimizer_utils.h"
-      "${ONNXRUNTIME_ROOT}/core/optimizer/transpose_optimizer/ort_transpose_optimizer.cc"
-      "${ONNXRUNTIME_ROOT}/core/optimizer/transpose_optimizer/ort_transpose_optimizer.h"
-      "${ONNXRUNTIME_ROOT}/core/optimizer/transpose_optimizer/transpose_optimizer.cc"
+      # files required for layout transformation
+      "${ONNXRUNTIME_ROOT}/core/optimizer/layout_transformation/layout_transformation.h"
+      "${ONNXRUNTIME_ROOT}/core/optimizer/layout_transformation/layout_transformation.cc"
+      "${ONNXRUNTIME_ROOT}/core/optimizer/layout_transformation/layout_transformation_potentially_added_ops.h"
+      # files required for transpose optimization post-layout transformation
+      "${ONNXRUNTIME_ROOT}/core/optimizer/transpose_optimization/optimizer_api.h"
+      "${ONNXRUNTIME_ROOT}/core/optimizer/transpose_optimization/onnx_transpose_optimization.h"
+      "${ONNXRUNTIME_ROOT}/core/optimizer/transpose_optimization/onnx_transpose_optimization.cc"
+      "${ONNXRUNTIME_ROOT}/core/optimizer/transpose_optimization/ort_optimizer_api_impl.cc"
+      "${ONNXRUNTIME_ROOT}/core/optimizer/transpose_optimization/ort_optimizer_utils.h"
+      "${ONNXRUNTIME_ROOT}/core/optimizer/transpose_optimization/ort_transpose_optimization.h"
+      "${ONNXRUNTIME_ROOT}/core/optimizer/transpose_optimization/ort_transpose_optimization.cc"
       "${ONNXRUNTIME_ROOT}/core/optimizer/utils.cc"
       "${ONNXRUNTIME_ROOT}/core/optimizer/utils.h"
     )
@@ -59,6 +65,8 @@ else()
     "${ONNXRUNTIME_ROOT}/core/optimizer/*.cc"
     "${ONNXRUNTIME_ROOT}/core/optimizer/compute_optimizer/*.h"
     "${ONNXRUNTIME_ROOT}/core/optimizer/compute_optimizer/*.cc"
+    "${ONNXRUNTIME_ROOT}/core/optimizer/layout_transformation/*.h"
+    "${ONNXRUNTIME_ROOT}/core/optimizer/layout_transformation/*.cc"
     "${ONNXRUNTIME_ROOT}/core/optimizer/qdq_transformer/*.h"
     "${ONNXRUNTIME_ROOT}/core/optimizer/qdq_transformer/*.cc"
     "${ONNXRUNTIME_ROOT}/core/optimizer/qdq_transformer/selectors_actions/*.h"
@@ -67,8 +75,8 @@ else()
     "${ONNXRUNTIME_ROOT}/core/optimizer/qdq_transformer/selectors_actions/shared/utils.cc"
     "${ONNXRUNTIME_ROOT}/core/optimizer/selectors_actions/*.h"
     "${ONNXRUNTIME_ROOT}/core/optimizer/selectors_actions/*.cc"
-    "${ONNXRUNTIME_ROOT}/core/optimizer/transpose_optimizer/*.h"
-    "${ONNXRUNTIME_ROOT}/core/optimizer/transpose_optimizer/*.cc"
+    "${ONNXRUNTIME_ROOT}/core/optimizer/transpose_optimization/*.h"
+    "${ONNXRUNTIME_ROOT}/core/optimizer/transpose_optimization/*.cc"
   )
 endif()
 

csharp/src/Microsoft.ML.OnnxRuntime/OrtValue.shared.cs

@@ -261,7 +261,6 @@ public ReadOnlySpan<byte> GetStringElementAsSpan(int index)
             }
         }
 
-
         /// <summary>
         /// Convenience method to obtain all string tensor elements as a string array.
         /// </summary>
@@ -774,6 +773,31 @@ public void FillStringTensorElement(ReadOnlyMemory<char> rom, int index)
             FillStringTensorElement(rom.Span, index);
         }
 
+        /// <summary>
+        /// This API resizes String Tensor element to the requested amount of bytes (UTF-8)
+        /// and copies the bytes from the supplied ReadOnlySpan into the native tensor memory (resized buffer).
+        /// 
+        /// The API is useful for quick loading of utf8 data into the native tensor memory.
+        /// </summary>
+        /// <param name="utf8Bytes">read only span of bytes</param>
+        /// <param name="index">flat index of the element in the string tensor</param>
+        public void FillStringTensorElement(ReadOnlySpan<byte> utf8Bytes, int index)
+        {
+            NativeApiStatus.VerifySuccess(NativeMethods.OrtGetResizedStringTensorElementBuffer(Handle,
+                                  (UIntPtr)index, (UIntPtr)utf8Bytes.Length, out IntPtr buffer));
+
+            if (utf8Bytes.Length == 0)
+            {
+                return;
+            }
+
+            unsafe
+            {
+                var destSpan = new Span<byte>(buffer.ToPointer(), utf8Bytes.Length);
+                utf8Bytes.CopyTo(destSpan);
+            }
+        }
+
         /// <summary>
         /// Creates an OrtValue that contains a string tensor.
         /// String tensors are always allocated on CPU.
@@ -953,7 +977,7 @@ protected virtual void Dispose(bool disposing)
 
             Debug.Assert(_handle != IntPtr.Zero);
             NativeMethods.OrtReleaseValue(_handle);
-           _handle = IntPtr.Zero;
+            _handle = IntPtr.Zero;
             _disposed = true;
         }
 

csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/TestDataLoader.cs

@@ -571,7 +571,11 @@ internal static NamedOnnxValue CreateNamedOnnxValueFromStringTensor(IList<Google
             string[] strArray = new string[strings.Count];
             for (int i = 0; i < strings.Count; ++i)
             {
-                strArray[i] = System.Text.Encoding.UTF8.GetString(strings[i].ToByteArray());
+#if NET6_0_OR_GREATER
+                strArray[i] = Encoding.UTF8.GetString(strings[i].Span);
+#else
+                strArray[i] = Encoding.UTF8.GetString(strings[i].ToByteArray());
+#endif
             }
 
             var dt = new DenseTensor<string>(strArray, dimensions);
@@ -585,8 +589,7 @@ internal static OrtValue CreateOrtValueFromStringTensor(IList<Google.Protobuf.By
             {
                 for (int i = 0; i < strings.Count; ++i)
                 {
-                    var str = Encoding.UTF8.GetString(strings[i].ToByteArray());
-                    ortValue.FillStringTensorElement(str.AsSpan(), i);
+                    ortValue.FillStringTensorElement(strings[i].Span, i);
                 }
                 return ortValue;
             }

include/onnxruntime/core/session/onnxruntime_cxx_api.h

@@ -210,6 +210,74 @@ struct BFloat16_t {
 
 static_assert(sizeof(BFloat16_t) == sizeof(uint16_t), "Sizes must match");
 
+/** \brief float8e4m3fn (Float8 Floating Point) data type
+ * \details It is necessary for type dispatching to make use of C++ API
+ * The type is implicitly convertible to/from uint8_t.
+ * See https://onnx.ai/onnx/technical/float8.html for further details.
+ */
+struct Float8E4M3FN_t {
+  uint8_t value;
+  constexpr Float8E4M3FN_t() noexcept : value(0) {}
+  constexpr Float8E4M3FN_t(uint8_t v) noexcept : value(v) {}
+  constexpr operator uint8_t() const noexcept { return value; }
+  // nan values are treated like any other value for operator ==, !=
+  constexpr bool operator==(const Float8E4M3FN_t& rhs) const noexcept { return value == rhs.value; };
+  constexpr bool operator!=(const Float8E4M3FN_t& rhs) const noexcept { return value != rhs.value; };
+};
+
+static_assert(sizeof(Float8E4M3FN_t) == sizeof(uint8_t), "Sizes must match");
+
+/** \brief float8e4m3fnuz (Float8 Floating Point) data type
+ * \details It is necessary for type dispatching to make use of C++ API
+ * The type is implicitly convertible to/from uint8_t.
+ * See https://onnx.ai/onnx/technical/float8.html for further details.
+ */
+struct Float8E4M3FNUZ_t {
+  uint8_t value;
+  constexpr Float8E4M3FNUZ_t() noexcept : value(0) {}
+  constexpr Float8E4M3FNUZ_t(uint8_t v) noexcept : value(v) {}
+  constexpr operator uint8_t() const noexcept { return value; }
+  // nan values are treated like any other value for operator ==, !=
+  constexpr bool operator==(const Float8E4M3FNUZ_t& rhs) const noexcept { return value == rhs.value; };
+  constexpr bool operator!=(const Float8E4M3FNUZ_t& rhs) const noexcept { return value != rhs.value; };
+};
+
+static_assert(sizeof(Float8E4M3FNUZ_t) == sizeof(uint8_t), "Sizes must match");
+
+/** \brief float8e5m2 (Float8 Floating Point) data type
+ * \details It is necessary for type dispatching to make use of C++ API
+ * The type is implicitly convertible to/from uint8_t.
+ * See https://onnx.ai/onnx/technical/float8.html for further details.
+ */
+struct Float8E5M2_t {
+  uint8_t value;
+  constexpr Float8E5M2_t() noexcept : value(0) {}
+  constexpr Float8E5M2_t(uint8_t v) noexcept : value(v) {}
+  constexpr operator uint8_t() const noexcept { return value; }
+  // nan values are treated like any other value for operator ==, !=
+  constexpr bool operator==(const Float8E5M2_t& rhs) const noexcept { return value == rhs.value; };
+  constexpr bool operator!=(const Float8E5M2_t& rhs) const noexcept { return value != rhs.value; };
+};
+
+static_assert(sizeof(Float8E5M2_t) == sizeof(uint8_t), "Sizes must match");
+
+/** \brief float8e5m2fnuz (Float8 Floating Point) data type
+ * \details It is necessary for type dispatching to make use of C++ API
+ * The type is implicitly convertible to/from uint8_t.
+ * See https://onnx.ai/onnx/technical/float8.html for further details.
+ */
+struct Float8E5M2FNUZ_t {
+  uint8_t value;
+  constexpr Float8E5M2FNUZ_t() noexcept : value(0) {}
+  constexpr Float8E5M2FNUZ_t(uint8_t v) noexcept : value(v) {}
+  constexpr operator uint8_t() const noexcept { return value; }
+  // nan values are treated like any other value for operator ==, !=
+  constexpr bool operator==(const Float8E5M2FNUZ_t& rhs) const noexcept { return value == rhs.value; };
+  constexpr bool operator!=(const Float8E5M2FNUZ_t& rhs) const noexcept { return value != rhs.value; };
+};
+
+static_assert(sizeof(Float8E5M2FNUZ_t) == sizeof(uint8_t), "Sizes must match");
+
 namespace detail {
 // This is used internally by the C++ API. This macro is to make it easy to generate overloaded methods for all of the various OrtRelease* functions for every Ort* type
 // This can't be done in the C API since C doesn't have function overloading.

include/onnxruntime/core/session/onnxruntime_cxx_inline.h

@@ -114,6 +114,23 @@ struct TypeToTensorType<bool> {
   static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL;
 };
 
+template <>
+struct TypeToTensorType<Float8E4M3FN_t> {
+  static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FN;
+};
+template <>
+struct TypeToTensorType<Float8E4M3FNUZ_t> {
+  static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FNUZ;
+};
+template <>
+struct TypeToTensorType<Float8E5M2_t> {
+  static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2;
+};
+template <>
+struct TypeToTensorType<Float8E5M2FNUZ_t> {
+  static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2FNUZ;
+};
+
 inline MemoryAllocation::MemoryAllocation(OrtAllocator* allocator, void* p, size_t size)
     : allocator_(allocator), p_(p), size_(size) {
 }

include/onnxruntime/core/session/onnxruntime_lite_custom_op.h

@@ -420,6 +420,10 @@ struct OrtLiteCustomOp : public OrtCustomOp {
   CREATE_TUPLE(uint64_t)
   CREATE_TUPLE(std::string)
   CREATE_TUPLE_INPUT(std::string_view)
+  CREATE_TUPLE(Ort::Float8E4M3FN_t)
+  CREATE_TUPLE(Ort::Float8E4M3FNUZ_t)
+  CREATE_TUPLE(Ort::Float8E5M2_t)
+  CREATE_TUPLE(Ort::Float8E5M2FNUZ_t)
 
   // ParseArgs ...
   template <typename... Ts>
@@ -493,6 +497,10 @@ struct OrtLiteCustomOp : public OrtCustomOp {
   PARSE_ARGS(uint64_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64)
   PARSE_ARGS(std::string, ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING)
   PARSE_ARGS(std::string_view, ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING)  // todo - remove string_view output
+  PARSE_ARGS(Ort::Float8E4M3FN_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FN)
+  PARSE_ARGS(Ort::Float8E4M3FNUZ_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FNUZ)
+  PARSE_ARGS(Ort::Float8E5M2_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2)
+  PARSE_ARGS(Ort::Float8E5M2FNUZ_t, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2FNUZ)
 
   OrtLiteCustomOp(const char* op_name,
                   const char* execution_provider) : op_name_(op_name),

js/web/script/test-runner-cli.ts

@@ -57,6 +57,7 @@ async function main() {
   const DEFAULT_OPSET_VERSIONS = fs.readdirSync(TEST_DATA_MODEL_NODE_ROOT, {withFileTypes: true})
                                      .filter(dir => dir.isDirectory() && dir.name.startsWith('opset'))
                                      .map(dir => dir.name.slice(5));
+  const MAX_OPSET_VERSION = Math.max(...DEFAULT_OPSET_VERSIONS.map(v => Number.parseInt(v, 10)));
 
   const FILE_CACHE_ENABLED = args.fileCache;         // whether to enable file cache
   const FILE_CACHE_MAX_FILE_SIZE = 1 * 1024 * 1024;  // The max size of the file that will be put into file cache
@@ -378,6 +379,7 @@ async function main() {
       // field 'verbose' and 'backend' is not set
       for (const test of tests) {
         test.backend = backend;
+        test.opsets = test.opsets || [{domain: '', version: MAX_OPSET_VERSION}];
       }
       npmlog.verbose('TestRunnerCli.Init.Op', 'Finished preparing test data.');
       npmlog.verbose('TestRunnerCli.Init.Op', '===============================================================');

js/web/test/data/ops/conv.jsonc

@@ -264,7 +264,10 @@
               20.0,
               21.0,
               22.0,
-              23.0
+              23.0,
+              0,
+              0,
+              0
             ],
             "dims": [1, 3, 3, 3],
             "type": "float32"

js/web/test/data/ops/leaky-relu.jsonc

@@ -8,14 +8,14 @@
         "name": "T[0]",
         "inputs": [
           {
-            "data": [-1.0, 0.0, 1.0, 1.0, -2.0],
+            "data": [-1.0, 0.0, 1.0, -2.0],
             "dims": [1, 4],
             "type": "float32"
           }
         ],
         "outputs": [
           {
-            "data": [-0.01, 0.0, 1.0, 1.0],
+            "data": [-0.01, 0.0, 1.0, -0.02],
             "dims": [1, 4],
             "type": "float32"
           }

js/web/test/data/ops/pad-big.jsonc

@@ -2,6 +2,7 @@
   {
     "name": "constant 2D",
     "operator": "Pad",
+    "opsets": [{"domain": "", "version": 10}],
     "attributes": [
       { "name": "mode", "data": "reflect", "type": "string" },
       { "name": "pads", "data": [0, 0, 1, 1, 0, 0, 1, 1], "type": "ints" }

js/web/test/data/ops/pad.jsonc

@@ -2,6 +2,7 @@
   {
     "name": "constant 2D",
     "operator": "Pad",
+    "opsets": [{"domain": "", "version": 10}],
     "attributes": [
       { "name": "mode", "data": "constant", "type": "string" },
       { "name": "value", "data": 1.2, "type": "float" },
@@ -80,6 +81,7 @@
   {
     "name": "constant 3D",
     "operator": "Pad",
+    "opsets": [{"domain": "", "version": 10}],
     "attributes": [
       { "name": "mode", "data": "constant", "type": "string" },
       { "name": "value", "data": 2.3, "type": "float" },
@@ -414,6 +416,7 @@
   {
     "name": "Reflect 1D",
     "operator": "Pad",
+    "opsets": [{"domain": "", "version": 10}],
     "attributes": [
       { "name": "mode", "data": "reflect", "type": "string" },
       { "name": "pads", "data": [5, 7], "type": "ints" }
@@ -457,6 +460,7 @@
   {
     "name": "Reflect 2D",
     "operator": "Pad",
+    "opsets": [{"domain": "", "version": 10}],
     "attributes": [
       { "name": "mode", "data": "reflect", "type": "string" },
       { "name": "pads", "data": [3, 2, 2, 5], "type": "ints" }
@@ -548,6 +552,7 @@
   {
     "name": "Reflect 3D",
     "operator": "Pad",
+    "opsets": [{"domain": "", "version": 10}],
     "attributes": [
       { "name": "mode", "data": "reflect", "type": "string" },
       { "name": "pads", "data": [1, 2, 2, 2, 3, 1], "type": "ints" }
@@ -881,6 +886,7 @@
   {
     "name": "Edge 2D",
     "operator": "Pad",
+    "opsets": [{"domain": "", "version": 10}],
     "attributes": [
       { "name": "mode", "data": "edge", "type": "string" },
       { "name": "pads", "data": [3, 2, 2, 3], "type": "ints" }
@@ -958,6 +964,7 @@
   {
     "name": "Edge 3D",
     "operator": "Pad",
+    "opsets": [{"domain": "", "version": 10}],
     "attributes": [
       { "name": "mode", "data": "edge", "type": "string" },
       { "name": "pads", "data": [1, 2, 2, 2, 3, 1], "type": "ints" }