From efad5bbc5aed1717200d3e8f6ddd253394af4b99 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Sat, 9 Mar 2024 09:17:36 -0800 Subject: [PATCH] Replace some old file system calls with C++17 std::filesystem APIs. (#19196) ### Description 1. Replace some old file system calls to use C++17 std::filesystem APIs. 2. Remove tensorflow_C_PACKAGE_PATH cmake option, which was only used in onnxruntime_perf_test and the code is out of maintain. 3. Excludes onnx_test_runner and onnxruntime_perf_test from iOS build because C++17 filesystem library is not available there --- cmake/CMakeLists.txt | 15 +- cmake/onnxruntime_unittests.cmake | 269 +++++++++--------- onnxruntime/test/onnx/TestCase.cc | 136 ++++----- onnxruntime/test/onnx/TestCase.h | 18 +- onnxruntime/test/onnx/onnx_model_info.cc | 10 +- onnxruntime/test/onnx/onnx_model_info.h | 10 +- onnxruntime/test/onnx/testcase_request.cc | 2 +- onnxruntime/test/perftest/TFModelInfo.cc | 58 ---- onnxruntime/test/perftest/TFModelInfo.h | 32 --- .../test/perftest/command_args_parser.cc | 5 +- .../test/perftest/performance_runner.cc | 42 +-- .../test/perftest/test_configuration.h | 1 - onnxruntime/test/providers/cpu/model_tests.cc | 132 ++++----- winml/test/model/model_tests.cpp | 2 +- 14 files changed, 307 insertions(+), 425 deletions(-) delete mode 100644 onnxruntime/test/perftest/TFModelInfo.cc delete mode 100644 onnxruntime/test/perftest/TFModelInfo.h diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 3f919d7bf6e1..02b568abdf8d 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -118,7 +118,6 @@ option(onnxruntime_GCOV_COVERAGE "Compile with options necessary to run code cov option(onnxruntime_DONT_VECTORIZE "Do not vectorize operations in Eigen" OFF) option(onnxruntime_USE_FULL_PROTOBUF "Link to libprotobuf instead of libprotobuf-lite when this option is ON" OFF) -option(tensorflow_C_PACKAGE_PATH "Path to tensorflow C package installation dir") option(onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS "Enable operator implemented in language other than cpp" OFF) option(onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS "Dump debug information about node inputs and outputs when executing the model." OFF) cmake_dependent_option(onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS_ENABLE_DUMP_TO_SQLDB "Build dump debug information about node inputs and outputs with support for sql database." OFF "onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS" OFF) @@ -1254,17 +1253,15 @@ if (onnxruntime_USE_TVM) $) set(onnxruntime_tvm_libs onnxruntime_providers_tvm) - - # needs to link with stdc++fs in Linux - if (UNIX) - if (NOT APPLE) - set(FS_STDLIB stdc++fs) - endif() - endif() - list(APPEND onnxruntime_EXTERNAL_LIBRARIES tvm ${FS_STDLIB}) + list(APPEND onnxruntime_EXTERNAL_LIBRARIES tvm) list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES tvm) endif() +# needs to link with stdc++fs in Linux +if (UNIX AND "${CMAKE_C_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 9) + set(FS_STDLIB stdc++fs) +endif() +list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${FS_STDLIB}) # onnxruntime-extensions if (onnxruntime_USE_EXTENSIONS) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index b004054c616a..cce39ae0f5d4 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -1,6 +1,6 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -if (${CMAKE_SYSTEM_NAME} STREQUAL "iOS") +if (IOS) find_package(XCTest REQUIRED) endif() @@ -18,7 +18,7 @@ function(AddTest) cmake_parse_arguments(_UT "DYN" "TARGET" "LIBS;SOURCES;DEPENDS;TEST_ARGS" ${ARGN}) list(REMOVE_DUPLICATES _UT_SOURCES) - if (${CMAKE_SYSTEM_NAME} STREQUAL "iOS") + if (IOS) onnxruntime_add_executable(${_UT_TARGET} ${TEST_SRC_DIR}/xctest/orttestmain.m) else() onnxruntime_add_executable(${_UT_TARGET} ${_UT_SOURCES}) @@ -129,7 +129,7 @@ function(AddTest) endif() endif(onnxruntime_GENERATE_TEST_REPORTS) - if (${CMAKE_SYSTEM_NAME} STREQUAL "iOS") + if (IOS) # target_sources(${_UT_TARGET} PRIVATE ${TEST_SRC_DIR}/xctest/orttestmain.m) set_target_properties(${_UT_TARGET} PROPERTIES FOLDER "ONNXRuntimeTest" MACOSX_BUNDLE_BUNDLE_NAME ${_UT_TARGET} @@ -734,34 +734,37 @@ target_include_directories(onnxruntime_test_utils PUBLIC "${TEST_SRC_DIR}/util/i set_target_properties(onnxruntime_test_utils PROPERTIES FOLDER "ONNXRuntimeTest") source_group(TREE ${TEST_SRC_DIR} FILES ${onnxruntime_test_utils_src}) -set(onnx_test_runner_src_dir ${TEST_SRC_DIR}/onnx) -file(GLOB onnx_test_runner_common_srcs CONFIGURE_DEPENDS - ${onnx_test_runner_src_dir}/*.h - ${onnx_test_runner_src_dir}/*.cc) +if(NOT IOS) + set(onnx_test_runner_src_dir ${TEST_SRC_DIR}/onnx) + file(GLOB onnx_test_runner_common_srcs CONFIGURE_DEPENDS + ${onnx_test_runner_src_dir}/*.h + ${onnx_test_runner_src_dir}/*.cc) -list(REMOVE_ITEM onnx_test_runner_common_srcs ${onnx_test_runner_src_dir}/main.cc) + list(REMOVE_ITEM onnx_test_runner_common_srcs ${onnx_test_runner_src_dir}/main.cc) -onnxruntime_add_static_library(onnx_test_runner_common ${onnx_test_runner_common_srcs}) -if(MSVC) - target_compile_options(onnx_test_runner_common PRIVATE "$<$:SHELL:--compiler-options /utf-8>" - "$<$>:/utf-8>") -else() - target_compile_definitions(onnx_test_runner_common PUBLIC -DNSYNC_ATOMIC_CPP11) - target_include_directories(onnx_test_runner_common PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT}) - onnxruntime_add_include_to_target(onnx_test_runner_common nsync::nsync_cpp) -endif() -if (MSVC AND NOT CMAKE_SIZEOF_VOID_P EQUAL 8) - #TODO: fix the warnings, they are dangerous - target_compile_options(onnx_test_runner_common PRIVATE "/wd4244") -endif() -onnxruntime_add_include_to_target(onnx_test_runner_common onnxruntime_common onnxruntime_framework - onnxruntime_test_utils onnx onnx_proto re2::re2 flatbuffers::flatbuffers Boost::mp11 safeint_interface) + onnxruntime_add_static_library(onnx_test_runner_common ${onnx_test_runner_common_srcs}) + if(MSVC) + target_compile_options(onnx_test_runner_common PRIVATE "$<$:SHELL:--compiler-options /utf-8>" + "$<$>:/utf-8>") + else() + target_compile_definitions(onnx_test_runner_common PUBLIC -DNSYNC_ATOMIC_CPP11) + target_include_directories(onnx_test_runner_common PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT}) + onnxruntime_add_include_to_target(onnx_test_runner_common nsync::nsync_cpp) + endif() + if (MSVC AND NOT CMAKE_SIZEOF_VOID_P EQUAL 8) + #TODO: fix the warnings, they are dangerous + target_compile_options(onnx_test_runner_common PRIVATE "/wd4244") + endif() + onnxruntime_add_include_to_target(onnx_test_runner_common onnxruntime_common onnxruntime_framework + onnxruntime_test_utils onnx onnx_proto re2::re2 flatbuffers::flatbuffers Boost::mp11 safeint_interface) -add_dependencies(onnx_test_runner_common onnx_test_data_proto ${onnxruntime_EXTERNAL_DEPENDENCIES}) -target_include_directories(onnx_test_runner_common PRIVATE ${eigen_INCLUDE_DIRS} - ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT}) + add_dependencies(onnx_test_runner_common onnx_test_data_proto ${onnxruntime_EXTERNAL_DEPENDENCIES}) + target_include_directories(onnx_test_runner_common PRIVATE ${eigen_INCLUDE_DIRS} + ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT}) -set_target_properties(onnx_test_runner_common PROPERTIES FOLDER "ONNXRuntimeTest") + set_target_properties(onnx_test_runner_common PROPERTIES FOLDER "ONNXRuntimeTest") + set(onnx_test_runner_common_lib onnx_test_runner_common) +endif() set(all_tests ${onnxruntime_test_common_src} ${onnxruntime_test_ir_src} ${onnxruntime_test_optimizer_src} ${onnxruntime_test_framework_src} ${onnxruntime_test_providers_src} ${onnxruntime_test_quantiztion_src}) @@ -820,6 +823,15 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten") "${TEST_SRC_DIR}/providers/cpu/tensor/grid_sample_test.cc") endif() +if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten" OR IOS) + # Because we do not run these model tests in our web or iOS CI build pipelines, and some test code uses C++17 + # filesystem functions that are not available in the iOS version we target. + message("Disable model tests in onnxruntime_test_all") + list(REMOVE_ITEM all_tests + "${TEST_SRC_DIR}/providers/cpu/model_tests.cc" + ) +endif() + set(test_all_args) if (onnxruntime_USE_TENSORRT) # TRT EP CI takes much longer time when updating to TRT 8.2 @@ -837,7 +849,7 @@ AddTest( TARGET onnxruntime_test_all SOURCES ${all_tests} ${onnxruntime_unittest_main_src} LIBS - onnx_test_runner_common ${onnxruntime_test_providers_libs} ${onnxruntime_test_common_libs} + ${onnx_test_runner_common_lib} ${onnxruntime_test_providers_libs} ${onnxruntime_test_common_libs} onnx_test_data_proto DEPENDS ${all_dependencies} TEST_ARGS ${test_all_args} @@ -875,7 +887,7 @@ endif() # the default logger tests conflict with the need to have an overall default logger # so skip in this type of target_compile_definitions(onnxruntime_test_all PUBLIC -DSKIP_DEFAULT_LOGGER_TESTS) -if (CMAKE_SYSTEM_NAME STREQUAL "iOS") +if (IOS) target_compile_definitions(onnxruntime_test_all_xc PUBLIC -DSKIP_DEFAULT_LOGGER_TESTS) endif() if(onnxruntime_RUN_MODELTEST_IN_DEBUG_MODE) @@ -1046,45 +1058,42 @@ if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS) list(APPEND onnx_test_libs onnxruntime_language_interop onnxruntime_pyop) endif() -onnxruntime_add_executable(onnx_test_runner ${onnx_test_runner_src_dir}/main.cc) -if(MSVC) - target_compile_options(onnx_test_runner PRIVATE "$<$:SHELL:--compiler-options /utf-8>" - "$<$>:/utf-8>") -endif() -if(${CMAKE_SYSTEM_NAME} STREQUAL "iOS") - set_target_properties(onnx_test_runner PROPERTIES - XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO" - ) -endif() -if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten") - if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) - set_target_properties(onnx_test_runner PROPERTIES LINK_FLAGS "-s NODERAWFS=1 -s ALLOW_MEMORY_GROWTH=1 -s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1") - else() - set_target_properties(onnx_test_runner PROPERTIES LINK_FLAGS "-s NODERAWFS=1 -s ALLOW_MEMORY_GROWTH=1") - endif() -endif() +if (NOT IOS) + onnxruntime_add_executable(onnx_test_runner ${onnx_test_runner_src_dir}/main.cc) + if(MSVC) + target_compile_options(onnx_test_runner PRIVATE "$<$:SHELL:--compiler-options /utf-8>" + "$<$>:/utf-8>") + endif() + if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten") + if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS) + set_target_properties(onnx_test_runner PROPERTIES LINK_FLAGS "-s NODERAWFS=1 -s ALLOW_MEMORY_GROWTH=1 -s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1") + else() + set_target_properties(onnx_test_runner PROPERTIES LINK_FLAGS "-s NODERAWFS=1 -s ALLOW_MEMORY_GROWTH=1") + endif() + endif() -target_link_libraries(onnx_test_runner PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs} nlohmann_json::nlohmann_json) -target_include_directories(onnx_test_runner PRIVATE ${ONNXRUNTIME_ROOT}) -if (onnxruntime_USE_ROCM) - target_include_directories(onnx_test_runner PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/onnxruntime ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/orttraining) -endif() -if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) - target_link_libraries(onnx_test_runner PRIVATE Python::Python) -endif() -set_target_properties(onnx_test_runner PROPERTIES FOLDER "ONNXRuntimeTest") + target_link_libraries(onnx_test_runner PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs} nlohmann_json::nlohmann_json) + target_include_directories(onnx_test_runner PRIVATE ${ONNXRUNTIME_ROOT}) + if (onnxruntime_USE_ROCM) + target_include_directories(onnx_test_runner PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/onnxruntime ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/orttraining) + endif() + if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) + target_link_libraries(onnx_test_runner PRIVATE Python::Python) + endif() + set_target_properties(onnx_test_runner PROPERTIES FOLDER "ONNXRuntimeTest") -if (onnxruntime_USE_TVM) - if (WIN32) - target_link_options(onnx_test_runner PRIVATE "/STACK:4000000") - endif() -endif() + if (onnxruntime_USE_TVM) + if (WIN32) + target_link_options(onnx_test_runner PRIVATE "/STACK:4000000") + endif() + endif() -install(TARGETS onnx_test_runner - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - BUNDLE DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + install(TARGETS onnx_test_runner + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + BUNDLE DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +endif() if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) if(onnxruntime_BUILD_BENCHMARKS) @@ -1165,90 +1174,80 @@ endif() if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) - #perf test runner - set(onnxruntime_perf_test_src_dir ${TEST_SRC_DIR}/perftest) - set(onnxruntime_perf_test_src_patterns - "${onnxruntime_perf_test_src_dir}/*.cc" - "${onnxruntime_perf_test_src_dir}/*.h") + if(NOT IOS) + #perf test runner + set(onnxruntime_perf_test_src_dir ${TEST_SRC_DIR}/perftest) + set(onnxruntime_perf_test_src_patterns + "${onnxruntime_perf_test_src_dir}/*.cc" + "${onnxruntime_perf_test_src_dir}/*.h") - if(WIN32) - list(APPEND onnxruntime_perf_test_src_patterns - "${onnxruntime_perf_test_src_dir}/windows/*.cc" - "${onnxruntime_perf_test_src_dir}/windows/*.h" ) - else () - list(APPEND onnxruntime_perf_test_src_patterns - "${onnxruntime_perf_test_src_dir}/posix/*.cc" - "${onnxruntime_perf_test_src_dir}/posix/*.h" ) - endif() + if(WIN32) + list(APPEND onnxruntime_perf_test_src_patterns + "${onnxruntime_perf_test_src_dir}/windows/*.cc" + "${onnxruntime_perf_test_src_dir}/windows/*.h" ) + else () + list(APPEND onnxruntime_perf_test_src_patterns + "${onnxruntime_perf_test_src_dir}/posix/*.cc" + "${onnxruntime_perf_test_src_dir}/posix/*.h" ) + endif() - file(GLOB onnxruntime_perf_test_src CONFIGURE_DEPENDS - ${onnxruntime_perf_test_src_patterns} - ) - onnxruntime_add_executable(onnxruntime_perf_test ${onnxruntime_perf_test_src} ${ONNXRUNTIME_ROOT}/core/platform/path_lib.cc) - if(MSVC) - target_compile_options(onnxruntime_perf_test PRIVATE "$<$:SHELL:--compiler-options /utf-8>" + file(GLOB onnxruntime_perf_test_src CONFIGURE_DEPENDS + ${onnxruntime_perf_test_src_patterns} + ) + onnxruntime_add_executable(onnxruntime_perf_test ${onnxruntime_perf_test_src} ${ONNXRUNTIME_ROOT}/core/platform/path_lib.cc) + if(MSVC) + target_compile_options(onnxruntime_perf_test PRIVATE "$<$:SHELL:--compiler-options /utf-8>" "$<$>:/utf-8>") - endif() - target_include_directories(onnxruntime_perf_test PRIVATE ${onnx_test_runner_src_dir} ${ONNXRUNTIME_ROOT} + endif() + target_include_directories(onnxruntime_perf_test PRIVATE ${onnx_test_runner_src_dir} ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${onnxruntime_graph_header} ${onnxruntime_exec_src_dir} ${CMAKE_CURRENT_BINARY_DIR}) - if (onnxruntime_USE_ROCM) - target_include_directories(onnxruntime_perf_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/onnxruntime ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/orttraining) - endif() - if (WIN32) - target_compile_options(onnxruntime_perf_test PRIVATE ${disabled_warnings}) - if (NOT DEFINED SYS_PATH_LIB) - set(SYS_PATH_LIB shlwapi) + if (onnxruntime_USE_ROCM) + target_include_directories(onnxruntime_perf_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/onnxruntime ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/orttraining) + endif() + if (WIN32) + target_compile_options(onnxruntime_perf_test PRIVATE ${disabled_warnings}) + if (NOT DEFINED SYS_PATH_LIB) + set(SYS_PATH_LIB shlwapi) + endif() endif() - endif() - if(${CMAKE_SYSTEM_NAME} STREQUAL "iOS") - set_target_properties(onnxruntime_perf_test PROPERTIES - XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO" - ) - endif() - if (onnxruntime_BUILD_SHARED_LIB) - #It will dynamically link to onnxruntime. So please don't add onxruntime_graph/onxruntime_framework/... here. - #onnxruntime_common is kind of ok because it is thin, tiny and totally stateless. - set(onnxruntime_perf_test_libs + if (onnxruntime_BUILD_SHARED_LIB) + #It will dynamically link to onnxruntime. So please don't add onxruntime_graph/onxruntime_framework/... here. + #onnxruntime_common is kind of ok because it is thin, tiny and totally stateless. + set(onnxruntime_perf_test_libs onnx_test_runner_common onnxruntime_test_utils onnxruntime_common onnxruntime onnxruntime_flatbuffers onnx_test_data_proto ${onnxruntime_EXTERNAL_LIBRARIES} ${GETOPT_LIB_WIDE} ${SYS_PATH_LIB} ${CMAKE_DL_LIBS}) - if(NOT WIN32) - list(APPEND onnxruntime_perf_test_libs nsync::nsync_cpp) - if(onnxruntime_USE_SNPE) - list(APPEND onnxruntime_perf_test_libs onnxruntime_providers_snpe) + if(NOT WIN32) + list(APPEND onnxruntime_perf_test_libs nsync::nsync_cpp) + if(onnxruntime_USE_SNPE) + list(APPEND onnxruntime_perf_test_libs onnxruntime_providers_snpe) + endif() endif() + if (CMAKE_SYSTEM_NAME STREQUAL "Android") + list(APPEND onnxruntime_perf_test_libs ${android_shared_libs}) + endif() + target_link_libraries(onnxruntime_perf_test PRIVATE ${onnxruntime_perf_test_libs} Threads::Threads) + if(WIN32) + target_link_libraries(onnxruntime_perf_test PRIVATE debug dbghelp advapi32) + endif() + else() + target_link_libraries(onnxruntime_perf_test PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs}) endif() - if (CMAKE_SYSTEM_NAME STREQUAL "Android") - list(APPEND onnxruntime_perf_test_libs ${android_shared_libs}) - endif() - target_link_libraries(onnxruntime_perf_test PRIVATE ${onnxruntime_perf_test_libs} Threads::Threads) - if(WIN32) - target_link_libraries(onnxruntime_perf_test PRIVATE debug dbghelp advapi32) - endif() - if(tensorflow_C_PACKAGE_PATH) - target_include_directories(onnxruntime_perf_test PRIVATE ${tensorflow_C_PACKAGE_PATH}/include) - target_link_directories(onnxruntime_perf_test PRIVATE ${tensorflow_C_PACKAGE_PATH}/lib) - target_link_libraries(onnxruntime_perf_test PRIVATE tensorflow) - target_compile_definitions(onnxruntime_perf_test PRIVATE HAVE_TENSORFLOW) - endif() - else() - target_link_libraries(onnxruntime_perf_test PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs}) - endif() - set_target_properties(onnxruntime_perf_test PROPERTIES FOLDER "ONNXRuntimeTest") + set_target_properties(onnxruntime_perf_test PROPERTIES FOLDER "ONNXRuntimeTest") - if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS AND NOT onnxruntime_BUILD_SHARED_LIB) - target_link_libraries(onnxruntime_perf_test PRIVATE onnxruntime_language_interop onnxruntime_pyop) - endif() + if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS AND NOT onnxruntime_BUILD_SHARED_LIB) + target_link_libraries(onnxruntime_perf_test PRIVATE onnxruntime_language_interop onnxruntime_pyop) + endif() - if (onnxruntime_USE_TVM) - if (WIN32) - target_link_options(onnxruntime_perf_test PRIVATE "/STACK:4000000") + if (onnxruntime_USE_TVM) + if (WIN32) + target_link_options(onnxruntime_perf_test PRIVATE "/STACK:4000000") + endif() endif() endif() - # shared lib if (onnxruntime_BUILD_SHARED_LIB) onnxruntime_add_static_library(onnxruntime_mocked_allocator ${TEST_SRC_DIR}/util/test_allocator.cc) @@ -1303,7 +1302,7 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) target_compile_definitions(onnxruntime_shared_lib_test PRIVATE USE_DUMMY_EXA_DEMANGLE=1) endif() - if (CMAKE_SYSTEM_NAME STREQUAL "iOS") + if (IOS) add_custom_command( TARGET onnxruntime_shared_lib_test POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory @@ -1390,7 +1389,7 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP) target_compile_options(onnxruntime_mlas_test PRIVATE "$<$:SHELL:--compiler-options /wd26426>" "$<$>:/wd26426>") endif() - if(${CMAKE_SYSTEM_NAME} STREQUAL "iOS") + if(IOS) set_target_properties(onnxruntime_mlas_test PROPERTIES XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED "NO" ) @@ -1591,7 +1590,7 @@ if (NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten") DEPENDS ${all_dependencies} ) - if (CMAKE_SYSTEM_NAME STREQUAL "iOS") + if (IOS) add_custom_command( TARGET onnxruntime_customopregistration_test POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory diff --git a/onnxruntime/test/onnx/TestCase.cc b/onnxruntime/test/onnx/TestCase.cc index 57c206188373..09c8ae213bad 100644 --- a/onnxruntime/test/onnx/TestCase.cc +++ b/onnxruntime/test/onnx/TestCase.cc @@ -267,12 +267,12 @@ void LoopDataFile(int test_data_pb_fd, bool is_input, const TestModelInfo& model } // namespace #if !defined(ORT_MINIMAL_BUILD) -std::unique_ptr TestModelInfo::LoadOnnxModel(_In_ const PATH_CHAR_TYPE* model_url) { +std::unique_ptr TestModelInfo::LoadOnnxModel(const std::filesystem::path& model_url) { return std::make_unique(model_url); } #endif -std::unique_ptr TestModelInfo::LoadOrtModel(_In_ const PATH_CHAR_TYPE* model_url) { +std::unique_ptr TestModelInfo::LoadOrtModel(const std::filesystem::path& model_url) { return std::make_unique(model_url, true); } @@ -290,7 +290,7 @@ class OnnxTestCase : public ITestCase { mutable std::vector debuginfo_strings_; mutable onnxruntime::OrtMutex m_; - std::vector> test_data_dirs_; + std::vector test_data_dirs_; std::string GetDatasetDebugInfoString(size_t dataset_id) const override { std::lock_guard l(m_); @@ -343,7 +343,7 @@ class OnnxTestCase : public ITestCase { size_t GetDataCount() const override { return test_data_dirs_.size(); } const std::string& GetNodeName() const override { return model_info_->GetNodeName(); } - const PATH_CHAR_TYPE* GetModelUrl() const override { return model_info_->GetModelUrl(); } + const std::filesystem::path& GetModelUrl() const override { return model_info_->GetModelUrl(); } const std::string& GetTestCaseName() const override { return test_case_name_; } std::string GetTestCaseVersion() const override { return model_info_->GetNominalOpsetVersion(); } @@ -396,7 +396,14 @@ static std::string trim_str(const std::string& in) { return s; } -static bool read_config_file(const std::basic_string& path, std::map& fc) { +/** + * @brief Read a text file that each line is a key value pair separated by ':' + * @param path File path + * @param fc output key value pairs + * @return True, success. False, the file doesn't exist or could be read. + */ +static bool ReadConfigFile(const std::filesystem::path& path, std::map& fc) { + if (!std::filesystem::exists(path)) return false; std::ifstream infile(path); if (!infile.good()) { return false; @@ -474,10 +481,10 @@ void OnnxTestCase::LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, ORT_THROW("index out of bound"); } - PATH_STRING_TYPE test_data_pb = ConcatPathComponent( - test_data_dirs_[id], (is_input ? ORT_TSTR("inputs.pb") : ORT_TSTR("outputs.pb"))); + std::filesystem::path test_data_pb = + test_data_dirs_[id] / (is_input ? ORT_TSTR("inputs.pb") : ORT_TSTR("outputs.pb")); int test_data_pb_fd; - auto st = Env::Default().FileOpenRd(test_data_pb, test_data_pb_fd); + auto st = Env::Default().FileOpenRd(test_data_pb.string(), test_data_pb_fd); if (st.IsOK()) { // has an all-in-one input file std::ostringstream oss; { @@ -504,21 +511,23 @@ void OnnxTestCase::LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, std::vector test_data_pb_files; - const PATH_STRING_TYPE& dir_path = test_data_dirs_[id]; - LoopDir(dir_path, - [&test_data_pb_files, &dir_path, is_input](const PATH_CHAR_TYPE* filename, OrtFileType f_type) -> bool { - if (filename[0] == '.') return true; - if (f_type != OrtFileType::TYPE_REG) return true; - std::basic_string filename_str = filename; - if (!HasExtensionOf(filename_str, ORT_TSTR("pb"))) return true; - const std::basic_string file_prefix = - is_input ? ORT_TSTR("input_") : ORT_TSTR("output_"); - if (!filename_str.compare(0, file_prefix.length(), file_prefix)) { - std::basic_string p = ConcatPathComponent(dir_path, filename_str); - test_data_pb_files.push_back(p); - } - return true; - }); + std::filesystem::path dir_fs_path = test_data_dirs_[id]; + if (!std::filesystem::exists(dir_fs_path)) return; + + for (auto const& dir_entry : std::filesystem::directory_iterator(dir_fs_path)) { + if (!dir_entry.is_regular_file()) continue; + const std::filesystem::path& path = dir_entry.path(); + if (!path.filename().has_extension()) { + continue; + } + if (path.filename().extension().compare(ORT_TSTR(".pb")) != 0) continue; + const std::basic_string file_prefix = + is_input ? ORT_TSTR("input_") : ORT_TSTR("output_"); + auto filename_str = path.filename().native(); + if (filename_str.compare(0, file_prefix.length(), file_prefix) == 0) { + test_data_pb_files.push_back(path.native()); + } + } SortFileNames(test_data_pb_files); @@ -691,11 +700,13 @@ void OnnxTestCase::ConvertTestData(const ONNX_NAMESPACE::OptionalProto& test_dat OnnxTestCase::OnnxTestCase(const std::string& test_case_name, _In_ std::unique_ptr model, double default_per_sample_tolerance, double default_relative_per_sample_tolerance) : test_case_name_(test_case_name), model_info_(std::move(model)) { - std::basic_string test_case_dir = model_info_->GetDir(); - + std::filesystem::path test_case_dir = model_info_->GetDir(); + if (!std::filesystem::exists(test_case_dir)) { + ORT_THROW("test case dir doesn't exist"); + } // parse config - std::basic_string config_path = - ConcatPathComponent(test_case_dir, ORT_TSTR("config.txt")); + std::filesystem::path config_path = + test_case_dir / ORT_TSTR("config.txt"); /* Note: protobuf-lite doesn't support reading protobuf files as text-format. Config.txt is exactly that. That's the reason I've to parse the file in a different way to read the configs. Currently this affects 2 tests - fp16_tiny_yolov2 and fp16_inception_v1. It's not clear why we've to use protobuf @@ -705,7 +716,7 @@ OnnxTestCase::OnnxTestCase(const std::string& test_case_name, _In_ std::unique_p per_sample_tolerance_ = default_per_sample_tolerance; relative_per_sample_tolerance_ = default_relative_per_sample_tolerance; post_processing_ = false; - if (read_config_file(config_path, fc)) { + if (ReadConfigFile(config_path, fc)) { if (fc.count("per_sample_tolerance") > 0) { per_sample_tolerance_ = stod(fc["per_sample_tolerance"]); } @@ -716,16 +727,11 @@ OnnxTestCase::OnnxTestCase(const std::string& test_case_name, _In_ std::unique_p post_processing_ = fc["post_processing"] == "true"; } } - - LoopDir(test_case_dir, [&test_case_dir, this](const PATH_CHAR_TYPE* filename, OrtFileType f_type) -> bool { - if (filename[0] == '.') return true; - if (f_type == OrtFileType::TYPE_DIR) { - std::basic_string p = ConcatPathComponent(test_case_dir, filename); - test_data_dirs_.push_back(p); - debuginfo_strings_.push_back(ToUTF8String(p)); - } - return true; - }); + for (auto const& dir_entry : std::filesystem::directory_iterator(test_case_dir)) { + if (!dir_entry.is_directory()) continue; + test_data_dirs_.push_back(dir_entry.path()); + debuginfo_strings_.push_back(ToUTF8String(dir_entry.path().string())); + } } void LoadTests(const std::vector>& input_paths, @@ -737,20 +743,23 @@ void LoadTests(const std::vector>& input_paths const std::function)>& process_function) { std::vector> paths(input_paths); while (!paths.empty()) { - std::basic_string node_data_root_path = paths.back(); + std::filesystem::path node_data_root_path = paths.back(); paths.pop_back(); - std::basic_string my_dir_name = GetLastComponent(node_data_root_path); - LoopDir(node_data_root_path, [&](const PATH_CHAR_TYPE* filename, OrtFileType f_type) -> bool { - if (filename[0] == '.') return true; - if (f_type == OrtFileType::TYPE_DIR) { - std::basic_string p = ConcatPathComponent(node_data_root_path, filename); - paths.push_back(p); - return true; + if (!std::filesystem::exists(node_data_root_path)) continue; + std::filesystem::path my_dir_name = node_data_root_path.filename(); + for (auto const& dir_entry : std::filesystem::directory_iterator(node_data_root_path)) { + if (dir_entry.is_directory()) { + paths.push_back(dir_entry.path()); + continue; } - - std::basic_string filename_str = filename; - bool is_onnx_format = HasExtensionOf(filename_str, ORT_TSTR("onnx")); - bool is_ort_format = HasExtensionOf(filename_str, ORT_TSTR("ort")); + if (!dir_entry.is_regular_file()) continue; + std::filesystem::path filename_str = dir_entry.path().filename(); + if (filename_str.empty() || filename_str.native()[0] == ORT_TSTR('.')) { + // Ignore hidden files. + continue; + } + bool is_onnx_format = filename_str.has_extension() && (filename_str.extension().compare(ORT_TSTR(".onnx")) == 0); + bool is_ort_format = filename_str.has_extension() && (filename_str.extension().compare(ORT_TSTR(".ort")) == 0); bool is_valid_model = false; #if !defined(ORT_MINIMAL_BUILD) @@ -759,42 +768,40 @@ void LoadTests(const std::vector>& input_paths is_valid_model = is_valid_model || is_ort_format; if (!is_valid_model) - return true; + continue; - std::basic_string test_case_name = my_dir_name; + std::basic_string test_case_name = my_dir_name.native(); if (test_case_name.compare(0, 5, ORT_TSTR("test_")) == 0) test_case_name = test_case_name.substr(5); if (!whitelisted_test_cases.empty() && std::find(whitelisted_test_cases.begin(), whitelisted_test_cases.end(), test_case_name) == whitelisted_test_cases.end()) { - return true; + continue; } - if (disabled_tests.find(test_case_name) != disabled_tests.end()) return true; - - std::basic_string p = ConcatPathComponent(node_data_root_path, filename_str); + if (disabled_tests.find(test_case_name) != disabled_tests.end()) continue; std::unique_ptr model_info; if (is_onnx_format) { #if !defined(ORT_MINIMAL_BUILD) - model_info = TestModelInfo::LoadOnnxModel(p.c_str()); + model_info = TestModelInfo::LoadOnnxModel(dir_entry.path()); #else ORT_THROW("onnx model is not supported in this build"); #endif } else if (is_ort_format) { - model_info = TestModelInfo::LoadOrtModel(p.c_str()); + model_info = TestModelInfo::LoadOrtModel(dir_entry.path()); } else { ORT_NOT_IMPLEMENTED(ToUTF8String(filename_str), " is not supported"); } auto test_case_dir = model_info->GetDir(); - auto test_case_name_in_log = test_case_name + ORT_TSTR(" in ") + test_case_dir; + auto test_case_name_in_log = test_case_name + ORT_TSTR(" in ") + test_case_dir.native(); #if !defined(ORT_MINIMAL_BUILD) && !defined(USE_QNN) // to skip some models like *-int8 or *-qdq if ((reinterpret_cast(model_info.get()))->HasDomain(ONNX_NAMESPACE::AI_ONNX_TRAINING_DOMAIN) || (reinterpret_cast(model_info.get()))->HasDomain(ONNX_NAMESPACE::AI_ONNX_PREVIEW_TRAINING_DOMAIN)) { fprintf(stderr, "Skip test case:: %s %s\n", ToUTF8String(test_case_name_in_log).c_str(), " as it has training domain"); - return true; + continue; } #endif @@ -809,7 +816,7 @@ void LoadTests(const std::vector>& input_paths }); if (!has_test_data) { fprintf(stderr, "Skip test case:: %s %s\n", ToUTF8String(test_case_name_in_log).c_str(), " due to no test data"); - return true; + continue; } if (broken_tests) { @@ -820,7 +827,7 @@ void LoadTests(const std::vector>& input_paths (opset_version == TestModelInfo::unknown_version || iter->broken_opset_versions_.empty() || iter->broken_opset_versions_.find(opset_version) != iter->broken_opset_versions_.end())) { fprintf(stderr, "Skip test case:: %s %s\n", ToUTF8String(test_case_name_in_log).c_str(), " due to broken_tests"); - return true; + continue; } } @@ -829,7 +836,7 @@ void LoadTests(const std::vector>& input_paths std::string keyword = *iter2; if (ToUTF8String(test_case_name).find(keyword) != std::string::npos) { fprintf(stderr, "Skip test case:: %s %s\n", ToUTF8String(test_case_name_in_log).c_str(), " as it is in broken test keywords"); - return true; + continue; } } } @@ -841,8 +848,7 @@ void LoadTests(const std::vector>& input_paths tolerances.relative(tolerance_key)); fprintf(stdout, "Load Test Case: %s\n", ToUTF8String(test_case_name_in_log).c_str()); process_function(std::move(l)); - return true; - }); + } } } diff --git a/onnxruntime/test/onnx/TestCase.h b/onnxruntime/test/onnx/TestCase.h index 96b0b5f6f7c0..0cb92056d378 100644 --- a/onnxruntime/test/onnx/TestCase.h +++ b/onnxruntime/test/onnx/TestCase.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -31,7 +32,7 @@ class ITestCase { virtual void LoadTestData(size_t id, onnxruntime::test::HeapBuffer& b, std::unordered_map& name_data_map, bool is_input) const = 0; - virtual const PATH_CHAR_TYPE* GetModelUrl() const = 0; + virtual const std::filesystem::path& GetModelUrl() const = 0; virtual const std::string& GetNodeName() const = 0; virtual const ONNX_NAMESPACE::ValueInfoProto* GetInputInfoFromModel(size_t i) const = 0; virtual const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t i) const = 0; @@ -50,14 +51,9 @@ class ITestCase { class TestModelInfo { public: - virtual const PATH_CHAR_TYPE* GetModelUrl() const = 0; - virtual std::basic_string GetDir() const { - std::basic_string test_case_dir; - auto st = onnxruntime::GetDirNameFromFilePath(GetModelUrl(), test_case_dir); - if (!st.IsOK()) { - ORT_THROW("GetDirNameFromFilePath failed"); - } - return test_case_dir; + virtual const std::filesystem::path& GetModelUrl() const = 0; + virtual std::filesystem::path GetDir() const { + return GetModelUrl().parent_path(); } virtual const std::string& GetNodeName() const = 0; virtual const ONNX_NAMESPACE::ValueInfoProto* GetInputInfoFromModel(size_t i) const = 0; @@ -70,10 +66,10 @@ class TestModelInfo { virtual ~TestModelInfo() = default; #if !defined(ORT_MINIMAL_BUILD) - static std::unique_ptr LoadOnnxModel(_In_ const PATH_CHAR_TYPE* model_url); + static std::unique_ptr LoadOnnxModel(const std::filesystem::path& model_url); #endif - static std::unique_ptr LoadOrtModel(_In_ const PATH_CHAR_TYPE* model_url); + static std::unique_ptr LoadOrtModel(const std::filesystem::path& model_url); static const std::string unknown_version; }; diff --git a/onnxruntime/test/onnx/onnx_model_info.cc b/onnxruntime/test/onnx/onnx_model_info.cc index d6afa99382e6..f23012aee9fd 100644 --- a/onnxruntime/test/onnx/onnx_model_info.cc +++ b/onnxruntime/test/onnx/onnx_model_info.cc @@ -14,7 +14,7 @@ using namespace onnxruntime; -OnnxModelInfo::OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url, bool is_ort_model) +OnnxModelInfo::OnnxModelInfo(const std::filesystem::path& model_url, bool is_ort_model) : model_url_(model_url) { if (is_ort_model) { InitOrtModelInfo(model_url); @@ -38,7 +38,7 @@ static void RepeatedPtrFieldToVector(const ::google::protobuf::RepeatedPtrField< } } -void OnnxModelInfo::InitOnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url) { // parse model +void OnnxModelInfo::InitOnnxModelInfo(const std::filesystem::path& model_url) { // parse model int model_fd; auto st = Env::Default().FileOpenRd(model_url, model_fd); if (!st.IsOK()) { @@ -50,7 +50,9 @@ void OnnxModelInfo::InitOnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url) { / const bool parse_result = model_pb.ParseFromZeroCopyStream(&input) && input.GetErrno() == 0; if (!parse_result) { (void)Env::Default().FileClose(model_fd); - ORT_THROW("Failed to load model because protobuf parsing failed."); + std::ostringstream oss; + oss << "Failed to load model from " << model_url << " because protobuf parsing failed."; + ORT_THROW(oss.str()); } (void)Env::Default().FileClose(model_fd); { @@ -91,7 +93,7 @@ void OnnxModelInfo::InitOnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url) { / #endif // #if !defined(ORT_MINIMAL_BUILD) -void OnnxModelInfo::InitOrtModelInfo(_In_ const PATH_CHAR_TYPE* model_url) { +void OnnxModelInfo::InitOrtModelInfo(const std::filesystem::path& model_url) { std::vector bytes; size_t num_bytes = 0; const auto model_location = ToWideString(model_url); diff --git a/onnxruntime/test/onnx/onnx_model_info.h b/onnxruntime/test/onnx/onnx_model_info.h index a0aa27df64a9..48e297376aff 100644 --- a/onnxruntime/test/onnx/onnx_model_info.h +++ b/onnxruntime/test/onnx/onnx_model_info.h @@ -13,16 +13,16 @@ class OnnxModelInfo : public TestModelInfo { std::vector input_value_info_; std::vector output_value_info_; std::unordered_map domain_to_version_; - const std::basic_string model_url_; + const std::filesystem::path model_url_; #if !defined(ORT_MINIMAL_BUILD) - void InitOnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url); + void InitOnnxModelInfo(const std::filesystem::path& model_url); #endif - void InitOrtModelInfo(_In_ const PATH_CHAR_TYPE* model_url); + void InitOrtModelInfo(const std::filesystem::path& model_url); public: - OnnxModelInfo(_In_ const PATH_CHAR_TYPE* model_url, bool is_ort_model = false); + OnnxModelInfo(const std::filesystem::path& path, bool is_ort_model = false); bool HasDomain(const std::string& name) const { return domain_to_version_.find(name) != domain_to_version_.end(); } @@ -32,7 +32,7 @@ class OnnxModelInfo : public TestModelInfo { return iter == domain_to_version_.end() ? -1 : iter->second; } - const PATH_CHAR_TYPE* GetModelUrl() const override { return model_url_.c_str(); } + const std::filesystem::path& GetModelUrl() const override { return model_url_; } std::string GetNominalOpsetVersion() const override { return onnx_nominal_opset_vesion_; } const std::string& GetNodeName() const override { return node_name_; } diff --git a/onnxruntime/test/onnx/testcase_request.cc b/onnxruntime/test/onnx/testcase_request.cc index 9ca8273ac907..9d653571ca2e 100644 --- a/onnxruntime/test/onnx/testcase_request.cc +++ b/onnxruntime/test/onnx/testcase_request.cc @@ -36,7 +36,7 @@ bool TestCaseRequestContext::SetupSession() { ORT_TRY { const auto* test_case_name = test_case_.GetTestCaseName().c_str(); session_opts_.SetLogId(test_case_name); - Ort::Session session{env_, test_case_.GetModelUrl(), session_opts_}; + Ort::Session session{env_, test_case_.GetModelUrl().native().c_str(), session_opts_}; session_ = std::move(session); LOGF_DEFAULT(INFO, "Testing %s\n", test_case_name); return true; diff --git a/onnxruntime/test/perftest/TFModelInfo.cc b/onnxruntime/test/perftest/TFModelInfo.cc deleted file mode 100644 index 82f5359545b4..000000000000 --- a/onnxruntime/test/perftest/TFModelInfo.cc +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "TFModelInfo.h" - -#include - -#include - -std::unique_ptr TFModelInfo::Create(_In_ const PATH_CHAR_TYPE* model_url) { - std::unique_ptr model_info = std::make_unique(); - - model_info->model_url_ = model_url; - std::basic_string meta_file_path = model_url; - meta_file_path.append(ORT_TSTR(".meta")); - const onnxruntime::Env& env = onnxruntime::Env::Default(); - size_t len; - auto status = env.GetFileLength(meta_file_path.c_str(), len); - if (!status.IsOK()) { - ORT_THROW(status.ErrorMessage()); - } - std::string file_content; - file_content.resize(len); - auto buffer_span = gsl::make_span(&file_content[0], file_content.size()); - status = onnxruntime::Env::Default().ReadFileIntoBuffer(meta_file_path.c_str(), 0, len, buffer_span); - if (!status.IsOK()) { - ORT_THROW(status.ErrorMessage()); - } - // this string is not null terminated - std::istringstream is{file_content}; - - std::string line; - while (std::getline(is, line)) { - size_t line_len = 0; - if (!line.empty() && line.back() == '\n') { - line_len = line.length() - 1; - if (line_len > 0 && line[line_len - 1] == '\r') { - --line_len; - } - line.resize(line_len); - } - if (line.empty()) continue; - if (line.compare(0, 6, "input=") == 0) { - model_info->input_names_.push_back(line.substr(6)); - } else if (line.compare(0, 7, "output=") == 0) { - model_info->output_names_.push_back(line.substr(7)); - } else { - ORT_THROW("unknown line:", line.size()); - } - } - - return model_info; -} - -int TFModelInfo::GetInputCount() const { return static_cast(input_names_.size()); } -int TFModelInfo::GetOutputCount() const { return static_cast(output_names_.size()); } -const std::string& TFModelInfo::GetInputName(size_t i) const { return input_names_[i]; } -const std::string& TFModelInfo::GetOutputName(size_t i) const { return output_names_[i]; } diff --git a/onnxruntime/test/perftest/TFModelInfo.h b/onnxruntime/test/perftest/TFModelInfo.h deleted file mode 100644 index 2ca60010e300..000000000000 --- a/onnxruntime/test/perftest/TFModelInfo.h +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include "TestCase.h" -#include -#include - -class TFModelInfo : public TestModelInfo { - public: - const PATH_CHAR_TYPE* GetModelUrl() const override { return model_url_.c_str(); } - - const std::string& GetNodeName() const override { return node_name_; } - const ONNX_NAMESPACE::ValueInfoProto* GetInputInfoFromModel(size_t) const override { return nullptr; } - const ONNX_NAMESPACE::ValueInfoProto* GetOutputInfoFromModel(size_t) const override { return nullptr; } - - int GetInputCount() const override; - int GetOutputCount() const override; - const std::string& GetInputName(size_t i) const override; - const std::string& GetOutputName(size_t i) const override; - ~TFModelInfo() override = default; - - static std::unique_ptr Create(_In_ const PATH_CHAR_TYPE* model_url); - TFModelInfo() = default; - - private: - std::basic_string model_url_; - std::vector input_names_; - std::vector output_names_; - std::string node_name_; -}; diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index 729ad3436845..16c90c39f300 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -199,7 +199,7 @@ static bool ParseSessionConfigs(const std::string& configs_string, /*static*/ bool CommandLineParser::ParseArguments(PerformanceTestConfig& test_config, int argc, ORTCHAR_T* argv[]) { int ch; - while ((ch = getopt(argc, argv, ORT_TSTR("b:m:e:r:t:p:x:y:c:d:o:u:i:f:F:S:T:C:AMPIDZvhsqzn"))) != -1) { + while ((ch = getopt(argc, argv, ORT_TSTR("m:e:r:t:p:x:y:c:d:o:u:i:f:F:S:T:C:AMPIDZvhsqzn"))) != -1) { switch (ch) { case 'f': { std::basic_string dim_name; @@ -228,9 +228,6 @@ static bool ParseSessionConfigs(const std::string& configs_string, return false; } break; - case 'b': - test_config.backend = optarg; - break; case 'p': test_config.run_config.profile_file = optarg; break; diff --git a/onnxruntime/test/perftest/performance_runner.cc b/onnxruntime/test/perftest/performance_runner.cc index 37bf80c80e90..08d77008dc25 100644 --- a/onnxruntime/test/perftest/performance_runner.cc +++ b/onnxruntime/test/perftest/performance_runner.cc @@ -10,12 +10,8 @@ #include #include "TestCase.h" -#include "TFModelInfo.h" #include "utils.h" #include "ort_test_session.h" -#ifdef HAVE_TENSORFLOW -#include "tf_test_session.h" -#endif using onnxruntime::Status; // TODO: Temporary, while we bring up the threadpool impl... @@ -260,47 +256,25 @@ Status PerformanceRunner::ForkJoinRepeat() { } static std::unique_ptr CreateModelInfo(const PerformanceTestConfig& performance_test_config_) { - if (CompareCString(performance_test_config_.backend.c_str(), ORT_TSTR("ort")) == 0) { - const auto& file_path = performance_test_config_.model_info.model_file_path; + const auto& file_path = performance_test_config_.model_info.model_file_path; #if !defined(ORT_MINIMAL_BUILD) - if (HasExtensionOf(file_path, ORT_TSTR("onnx"))) { - return TestModelInfo::LoadOnnxModel(performance_test_config_.model_info.model_file_path.c_str()); - } -#endif - - if (HasExtensionOf(file_path, ORT_TSTR("ort"))) { - return TestModelInfo::LoadOrtModel(performance_test_config_.model_info.model_file_path.c_str()); - } - - ORT_NOT_IMPLEMENTED(ToUTF8String(file_path), " is not supported"); + if (HasExtensionOf(file_path, ORT_TSTR("onnx"))) { + return TestModelInfo::LoadOnnxModel(performance_test_config_.model_info.model_file_path.c_str()); } +#endif - if (CompareCString(performance_test_config_.backend.c_str(), ORT_TSTR("tf")) == 0) { - return TFModelInfo::Create(performance_test_config_.model_info.model_file_path.c_str()); + if (HasExtensionOf(file_path, ORT_TSTR("ort"))) { + return TestModelInfo::LoadOrtModel(performance_test_config_.model_info.model_file_path.c_str()); } - ORT_NOT_IMPLEMENTED(ToUTF8String(performance_test_config_.backend), " is not supported"); -} - -static std::unique_ptr CreateSession(Ort::Env& env, std::random_device& rd, - const PerformanceTestConfig& performance_test_config_, - const TestModelInfo& test_model_info) { - if (CompareCString(performance_test_config_.backend.c_str(), ORT_TSTR("ort")) == 0) { - return std::make_unique(env, rd, performance_test_config_, test_model_info); - } -#ifdef HAVE_TENSORFLOW - if (CompareCString(performance_test_config_.backend.c_str(), ORT_TSTR("tf")) == 0) { - return new TensorflowTestSession(rd, performance_test_config_, test_model_info); - } -#endif - ORT_NOT_IMPLEMENTED(ToUTF8String(performance_test_config_.backend), " is not supported"); + ORT_NOT_IMPLEMENTED(ToUTF8String(file_path), " is not supported"); } PerformanceRunner::PerformanceRunner(Ort::Env& env, const PerformanceTestConfig& test_config, std::random_device& rd) : performance_test_config_(test_config), test_model_info_(CreateModelInfo(test_config)) { session_create_start_ = std::chrono::high_resolution_clock::now(); - session_ = CreateSession(env, rd, test_config, *test_model_info_); + session_ = std::make_unique(env, rd, performance_test_config_, *test_model_info_); session_create_end_ = std::chrono::high_resolution_clock::now(); } diff --git a/onnxruntime/test/perftest/test_configuration.h b/onnxruntime/test/perftest/test_configuration.h index 74c8eb472cb3..70a6b12690d5 100644 --- a/onnxruntime/test/perftest/test_configuration.h +++ b/onnxruntime/test/perftest/test_configuration.h @@ -70,7 +70,6 @@ struct PerformanceTestConfig { ModelInfo model_info; MachineConfig machine_config; RunConfig run_config; - std::basic_string backend = ORT_TSTR("ort"); }; } // namespace perftest diff --git a/onnxruntime/test/providers/cpu/model_tests.cc b/onnxruntime/test/providers/cpu/model_tests.cc index 8128c170c521..746171737714 100644 --- a/onnxruntime/test/providers/cpu/model_tests.cc +++ b/onnxruntime/test/providers/cpu/model_tests.cc @@ -3,6 +3,12 @@ #include #include +#include +#include +#include +#include +#include +#include #include #include "core/session/onnxruntime_c_api.h" @@ -15,9 +21,6 @@ #include #include "default_providers.h" #include "test/onnx/TestCase.h" -#include -#include -#include #ifdef USE_DNNL #include "core/providers/dnnl/dnnl_provider_factory.h" @@ -47,7 +50,6 @@ #include "test/compare_ortvalue.h" #include "test/onnx/heap_buffer.h" #include "test/onnx/onnx_model_info.h" -#include "test/onnx/callback.h" #include "test/onnx/testcase_request.h" extern std::unique_ptr ort_env; @@ -378,46 +380,46 @@ TEST_P(ModelTest, Run) { } using ORT_STRING_VIEW = std::basic_string_view; -static ORT_STRING_VIEW opset7 = ORT_TSTR("opset7"); -static ORT_STRING_VIEW opset8 = ORT_TSTR("opset8"); -static ORT_STRING_VIEW opset9 = ORT_TSTR("opset9"); -static ORT_STRING_VIEW opset10 = ORT_TSTR("opset10"); -static ORT_STRING_VIEW opset11 = ORT_TSTR("opset11"); -static ORT_STRING_VIEW opset12 = ORT_TSTR("opset12"); -static ORT_STRING_VIEW opset13 = ORT_TSTR("opset13"); -static ORT_STRING_VIEW opset14 = ORT_TSTR("opset14"); -static ORT_STRING_VIEW opset15 = ORT_TSTR("opset15"); -static ORT_STRING_VIEW opset16 = ORT_TSTR("opset16"); -static ORT_STRING_VIEW opset17 = ORT_TSTR("opset17"); -static ORT_STRING_VIEW opset18 = ORT_TSTR("opset18"); +static constexpr ORT_STRING_VIEW opset7 = ORT_TSTR("opset7"); +static constexpr ORT_STRING_VIEW opset8 = ORT_TSTR("opset8"); +static constexpr ORT_STRING_VIEW opset9 = ORT_TSTR("opset9"); +static constexpr ORT_STRING_VIEW opset10 = ORT_TSTR("opset10"); +static constexpr ORT_STRING_VIEW opset11 = ORT_TSTR("opset11"); +static constexpr ORT_STRING_VIEW opset12 = ORT_TSTR("opset12"); +static constexpr ORT_STRING_VIEW opset13 = ORT_TSTR("opset13"); +static constexpr ORT_STRING_VIEW opset14 = ORT_TSTR("opset14"); +static constexpr ORT_STRING_VIEW opset15 = ORT_TSTR("opset15"); +static constexpr ORT_STRING_VIEW opset16 = ORT_TSTR("opset16"); +static constexpr ORT_STRING_VIEW opset17 = ORT_TSTR("opset17"); +static constexpr ORT_STRING_VIEW opset18 = ORT_TSTR("opset18"); // TODO: enable opset19 tests -// static ORT_STRING_VIEW opset19 = ORT_TSTR("opset19"); +// static constexpr ORT_STRING_VIEW opset19 = ORT_TSTR("opset19"); -static ORT_STRING_VIEW provider_name_cpu = ORT_TSTR("cpu"); -static ORT_STRING_VIEW provider_name_tensorrt = ORT_TSTR("tensorrt"); +static constexpr ORT_STRING_VIEW provider_name_cpu = ORT_TSTR("cpu"); +static constexpr ORT_STRING_VIEW provider_name_tensorrt = ORT_TSTR("tensorrt"); #ifdef USE_MIGRAPHX -static ORT_STRING_VIEW provider_name_migraphx = ORT_TSTR("migraphx"); +static constexpr ORT_STRING_VIEW provider_name_migraphx = ORT_TSTR("migraphx"); #endif -static ORT_STRING_VIEW provider_name_openvino = ORT_TSTR("openvino"); -static ORT_STRING_VIEW provider_name_cuda = ORT_TSTR("cuda"); +static constexpr ORT_STRING_VIEW provider_name_openvino = ORT_TSTR("openvino"); +static constexpr ORT_STRING_VIEW provider_name_cuda = ORT_TSTR("cuda"); #ifdef USE_ROCM -static ORT_STRING_VIEW provider_name_rocm = ORT_TSTR("rocm"); +static constexpr ORT_STRING_VIEW provider_name_rocm = ORT_TSTR("rocm"); #endif -static ORT_STRING_VIEW provider_name_dnnl = ORT_TSTR("dnnl"); +static constexpr ORT_STRING_VIEW provider_name_dnnl = ORT_TSTR("dnnl"); // For any non-Android system, NNAPI will only be used for ort model converter #if defined(USE_NNAPI) && defined(__ANDROID__) -static ORT_STRING_VIEW provider_name_nnapi = ORT_TSTR("nnapi"); +static constexpr ORT_STRING_VIEW provider_name_nnapi = ORT_TSTR("nnapi"); #endif #ifdef USE_RKNPU -static ORT_STRING_VIEW provider_name_rknpu = ORT_TSTR("rknpu"); +static constexpr ORT_STRING_VIEW provider_name_rknpu = ORT_TSTR("rknpu"); #endif #ifdef USE_ACL -static ORT_STRING_VIEW provider_name_acl = ORT_TSTR("acl"); +static constexpr ORT_STRING_VIEW provider_name_acl = ORT_TSTR("acl"); #endif #ifdef USE_ARMNN -static ORT_STRING_VIEW provider_name_armnn = ORT_TSTR("armnn"); +static constexpr ORT_STRING_VIEW provider_name_armnn = ORT_TSTR("armnn"); #endif -static ORT_STRING_VIEW provider_name_dml = ORT_TSTR("dml"); +static constexpr ORT_STRING_VIEW provider_name_dml = ORT_TSTR("dml"); ::std::vector<::std::basic_string> GetParameterStrings() { // Map key is provider name(CPU, CUDA, etc). Value is the ONNX node tests' opsets to run. @@ -615,7 +617,7 @@ ::std::vector<::std::basic_string> GetParameterStrings() { ORT_TSTR("SSD"), // needs to run symbolic shape inference shape first ORT_TSTR("size") // INVALID_ARGUMENT: Cannot find binding of given name: x }; - std::vector> paths; + std::vector paths; for (std::pair> kvp : provider_names) { // Setup ONNX node tests. The test data is preloaded on our CI build machines. @@ -644,7 +646,7 @@ ::std::vector<::std::basic_string> GetParameterStrings() { } #endif - ORT_STRING_VIEW provider_name = kvp.first; + const ORT_STRING_VIEW provider_name = kvp.first; std::unordered_set> all_disabled_tests(std::begin(immutable_broken_tests), std::end(immutable_broken_tests)); if (provider_name == provider_name_cuda) { @@ -699,45 +701,45 @@ ::std::vector<::std::basic_string> GetParameterStrings() { all_disabled_tests.insert(ORT_TSTR("fp16_tiny_yolov2")); while (!paths.empty()) { - std::basic_string node_data_root_path = paths.back(); + std::filesystem::path node_data_root_path = paths.back(); paths.pop_back(); - std::basic_string my_dir_name = GetLastComponent(node_data_root_path); - ORT_TRY { - LoopDir(node_data_root_path, [&](const ORTCHAR_T* filename, OrtFileType f_type) -> bool { - if (filename[0] == ORT_TSTR('.')) - return true; - if (f_type == OrtFileType::TYPE_DIR) { - std::basic_string p = ConcatPathComponent(node_data_root_path, filename); - paths.push_back(p); - return true; - } - std::basic_string filename_str = filename; - if (!HasExtensionOf(filename_str, ORT_TSTR("onnx"))) - return true; - std::basic_string test_case_name = my_dir_name; - if (test_case_name.compare(0, 5, ORT_TSTR("test_")) == 0) - test_case_name = test_case_name.substr(5); - if (all_disabled_tests.find(test_case_name) != all_disabled_tests.end()) - return true; + if (!std::filesystem::exists(node_data_root_path) || !std::filesystem::is_directory(node_data_root_path)) { + continue; + } + for (auto const& dir_entry : std::filesystem::directory_iterator(node_data_root_path)) { + if (dir_entry.is_directory()) { + paths.push_back(dir_entry.path()); + continue; + } + const std::filesystem::path& path = dir_entry.path(); + if (!path.has_filename() || path.filename().native().compare(0, 1, ORT_TSTR(".")) == 0) { + // Ignore hidden files. + continue; + } + if (path.filename().extension().compare(ORT_TSTR(".onnx")) != 0) { + // Ignore the files that are not ONNX models + continue; + } + std::basic_string test_case_name = path.parent_path().filename().native(); + if (test_case_name.compare(0, 5, ORT_TSTR("test_")) == 0) + test_case_name = test_case_name.substr(5); + if (all_disabled_tests.find(test_case_name) != all_disabled_tests.end()) + continue; #ifdef DISABLE_ML_OPS - auto starts_with = [](const std::basic_string& find_in, - const std::basic_string& find_what) { - return find_in.compare(0, find_what.size(), find_what) == 0; - }; - if (starts_with(test_case_name, ORT_TSTR("XGBoost_")) || starts_with(test_case_name, ORT_TSTR("coreml_")) || - starts_with(test_case_name, ORT_TSTR("scikit_")) || starts_with(test_case_name, ORT_TSTR("libsvm_"))) { - return true; - } + auto starts_with = [](const std::basic_string& find_in, + const std::basic_string& find_what) { + return find_in.compare(0, find_what.size(), find_what) == 0; + }; + if (starts_with(test_case_name, ORT_TSTR("XGBoost_")) || starts_with(test_case_name, ORT_TSTR("coreml_")) || + starts_with(test_case_name, ORT_TSTR("scikit_")) || starts_with(test_case_name, ORT_TSTR("libsvm_"))) { + continue; + } #endif - std::basic_ostringstream oss; - oss << provider_name << ORT_TSTR("_") << ConcatPathComponent(node_data_root_path, filename_str); - v.emplace_back(oss.str()); - return true; - }); + std::basic_ostringstream oss; + oss << provider_name << ORT_TSTR("_") << path.native(); + v.emplace_back(oss.str()); } - ORT_CATCH(const std::exception&) { - } // ignore non-exist dir } } return v; diff --git a/winml/test/model/model_tests.cpp b/winml/test/model/model_tests.cpp index f40f08ad2696..27d74d7d6b03 100644 --- a/winml/test/model/model_tests.cpp +++ b/winml/test/model/model_tests.cpp @@ -118,7 +118,7 @@ TEST_P(ModelTest, Run) { LearningModelDevice device = nullptr; LearningModelSession session = nullptr; LearningModelBinding binding = nullptr; - WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(m_testCase->GetModelUrl())); + WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(m_testCase->GetModelUrl().native())); WINML_EXPECT_NO_THROW(device = LearningModelDevice(m_deviceKind)); WINML_EXPECT_NO_THROW(session = LearningModelSession(model, device)); for (size_t i = 0; i < m_testCase->GetDataCount(); i++) {