Find portable_lib.so in pip package during cmake build (#5961)

Summary: * Rename `_portable_lib.cpython-3.<distribution info>.so` to `_portable_lib.so` so it can be found by CMake `find_library()`. This can be achieved by setting `SETUPTOOLS_EXT_SUFFIX`. * Since `executorch-config.cmake` is also being used to find installed libraries such as `executorch.a`, `xnnpack_backend.a`, add a condition to tell if `executorch-config.cmake` is being used in cmake-out or site-packages. Pull Request resolved: #5961 Reviewed By: metascroy Differential Revision: D64014291 Pulled By: larryliu0820 fbshipit-source-id: 2757f2883d3f836e9efd45676f792c12f742e63d
pytorch · Oct 8, 2024 · 7337f8e · 7337f8e
1 parent 03e4516
commit 7337f8e
Show file tree

Hide file tree

Showing 4 changed files with 117 additions and 72 deletions.
diff --git a/build/executorch-config.cmake b/build/executorch-config.cmake
@@ -9,85 +9,117 @@
 # is:
 #
 # find_package(executorch REQUIRED)
-
+# -------
+#
+# Finds the ExecuTorch library
+#
+# This will define the following variables:
+#
+#   EXECUTORCH_FOUND        -- True if the system has the Torch library
+#   EXECUTORCH_INCLUDE_DIRS -- The include directories for torch
+#   EXECUTORCH_LIBRARIES    -- Libraries to link against
+#
 cmake_minimum_required(VERSION 3.19)
 
-set(_root "${CMAKE_CURRENT_LIST_DIR}/../..")
-set(required_lib_list executorch executorch_core portable_kernels)
-foreach(lib ${required_lib_list})
-  set(lib_var "LIB_${lib}")
-  add_library(${lib} STATIC IMPORTED)
-  find_library(
-    ${lib_var} ${lib}
-    HINTS "${_root}"
-    CMAKE_FIND_ROOT_PATH_BOTH
+# Find prebuilt libportable_lib.so. If found, assuming current file is inside
+# a pip package:
+#     <site-packages>/executorch/executorch-config.cmake.
+# If not found, assuming current file is inside cmake-out:
+#     <cmake-out>/cmake/ExecuTorch/executorch-config.cmake
+find_library(_portable_lib_LIBRARY _portable_lib.so PATHS "${CMAKE_CURRENT_LIST_DIR}/extension/pybindings/")
+message(WARNING "${CMAKE_CURRENT_LIST_DIR}/extension/pybindings/")
+set(EXECUTORCH_LIBRARIES)
+if(_portable_lib_LIBRARY)
+  # Assuming current file is <site-packages>/executorch/executorch-config.cmake
+  message(WARNING "portable library is found")
+  list(APPEND EXECUTORCH_LIBRARIES _portable_lib)
+  add_library(_portable_lib STATIC IMPORTED)
+  set(EXECUTORCH_INCLUDE_DIRS ${CMAKE_CURRENT_LIST_DIR}/include)
+  set_target_properties(_portable_lib PROPERTIES
+    IMPORTED_LOCATION "${_portable_lib_LIBRARY}"
+    INTERFACE_INCLUDE_DIRECTORIES "${EXECUTORCH_INCLUDE_DIRS}"
+    CXX_STANDARD 17
   )
-  set_target_properties(${lib} PROPERTIES IMPORTED_LOCATION "${${lib_var}}")
-  target_include_directories(${lib} INTERFACE ${_root})
-endforeach()
+else()
+  # Assuming current file is <cmake-out>/cmake/ExecuTorch/executorch-config.cmake
+  message(WARNING "portable library is not found")
+  set(_root "${CMAKE_CURRENT_LIST_DIR}/../..")
+  set(required_lib_list executorch executorch_core portable_kernels)
+  foreach(lib ${required_lib_list})
+    set(lib_var "LIB_${lib}")
+    add_library(${lib} STATIC IMPORTED)
+    find_library(
+      ${lib_var} ${lib}
+      HINTS "${_root}"
+      CMAKE_FIND_ROOT_PATH_BOTH
+    )
+    set_target_properties(${lib} PROPERTIES IMPORTED_LOCATION "${${lib_var}}")
+    target_include_directories(${lib} INTERFACE ${_root})
+  endforeach()
 
-target_link_libraries(executorch INTERFACE executorch_core)
+  target_link_libraries(executorch INTERFACE executorch_core)
 
-if(CMAKE_BUILD_TYPE MATCHES "Debug")
-  set(FLATCCRT_LIB flatccrt_d)
-else()
-  set(FLATCCRT_LIB flatccrt)
-endif()
+  if(CMAKE_BUILD_TYPE MATCHES "Debug")
+    set(FLATCCRT_LIB flatccrt_d)
+  else()
+    set(FLATCCRT_LIB flatccrt)
+  endif()
 
-set(lib_list
-    etdump
-    bundled_program
-    extension_data_loader
-    ${FLATCCRT_LIB}
-    coremldelegate
-    mpsdelegate
-    qnn_executorch_backend
-    portable_ops_lib
-    extension_module
-    extension_module_static
-    extension_runner_util
-    extension_tensor
-    extension_threadpool
-    extension_training
-    xnnpack_backend
-    # Start XNNPACK Lib Deps
-    XNNPACK
-    microkernels-prod
-    kleidiai
-    # End XNNPACK Lib Deps
-    cpuinfo
-    pthreadpool
-    vulkan_backend
-    optimized_kernels
-    cpublas
-    eigen_blas
-    optimized_ops_lib
-    optimized_native_cpu_ops_lib
-    quantized_kernels
-    quantized_ops_lib
-    quantized_ops_aot_lib
-)
-foreach(lib ${lib_list})
-  # Name of the variable which stores result of the find_library search
-  set(lib_var "LIB_${lib}")
-  find_library(
-    ${lib_var} ${lib}
-    HINTS "${_root}"
-    CMAKE_FIND_ROOT_PATH_BOTH
+  set(lib_list
+      etdump
+      bundled_program
+      extension_data_loader
+      ${FLATCCRT_LIB}
+      coremldelegate
+      mpsdelegate
+      qnn_executorch_backend
+      portable_ops_lib
+      extension_module
+      extension_module_static
+      extension_runner_util
+      extension_tensor
+      extension_threadpool
+      extension_training
+      xnnpack_backend
+      # Start XNNPACK Lib Deps
+      XNNPACK
+      microkernels-prod
+      kleidiai
+      # End XNNPACK Lib Deps
+      cpuinfo
+      pthreadpool
+      vulkan_backend
+      optimized_kernels
+      cpublas
+      eigen_blas
+      optimized_ops_lib
+      optimized_native_cpu_ops_lib
+      quantized_kernels
+      quantized_ops_lib
+      quantized_ops_aot_lib
   )
-  if(NOT ${lib_var})
-    message("${lib} library is not found.
-            If needed rebuild with the proper options in CMakeLists.txt"
+  foreach(lib ${lib_list})
+    # Name of the variable which stores result of the find_library search
+    set(lib_var "LIB_${lib}")
+    find_library(
+      ${lib_var} ${lib}
+      HINTS "${_root}"
+      CMAKE_FIND_ROOT_PATH_BOTH
     )
-  else()
-    if("${lib}" STREQUAL "extension_module" AND (NOT CMAKE_TOOLCHAIN_IOS))
-      add_library(${lib} SHARED IMPORTED)
+    if(NOT ${lib_var})
+      message("${lib} library is not found.
+              If needed rebuild with the proper options in CMakeLists.txt"
+      )
     else()
-      # Building a share library on iOS requires code signing, so it's easier to
-      # keep all libs as static when CMAKE_TOOLCHAIN_IOS is used
-      add_library(${lib} STATIC IMPORTED)
+      if("${lib}" STREQUAL "extension_module" AND (NOT CMAKE_TOOLCHAIN_IOS))
+        add_library(${lib} SHARED IMPORTED)
+      else()
+        # Building a share library on iOS requires code signing, so it's easier to
+        # keep all libs as static when CMAKE_TOOLCHAIN_IOS is used
+        add_library(${lib} STATIC IMPORTED)
+      endif()
+      set_target_properties(${lib} PROPERTIES IMPORTED_LOCATION "${${lib_var}}")
+      target_include_directories(${lib} INTERFACE ${_root})
     endif()
-    set_target_properties(${lib} PROPERTIES IMPORTED_LOCATION "${${lib_var}}")
-    target_include_directories(${lib} INTERFACE ${_root})
-  endif()
-endforeach()
+  endforeach()
+endif()
diff --git a/build/packaging/env_var_script_linux.sh b/build/packaging/env_var_script_linux.sh
@@ -11,6 +11,10 @@
 # Enable pybindings so that users can execute ExecuTorch programs from python.
 export EXECUTORCH_BUILD_PYBIND=1
 
+# Override extension suffix to be ".so", skipping package info such as
+# "cpython-311-x86_64-linux-gnu"
+export SETUPTOOLS_EXT_SUFFIX=".so"
+
 # Ensure that CMAKE_ARGS is defined before referencing it. Defaults to empty
 # if not defined.
 export CMAKE_ARGS="${CMAKE_ARGS:-}"

diff --git a/build/packaging/env_var_script_m1.sh b/build/packaging/env_var_script_m1.sh
@@ -11,6 +11,10 @@
 # Enable pybindings so that users can execute ExecuTorch programs from python.
 export EXECUTORCH_BUILD_PYBIND=1
 
+# Override extension suffix to be ".so", skipping package info such as
+# "cpython-311-darwin"
+export SETUPTOOLS_EXT_SUFFIX=".so"
+
 # Ensure that CMAKE_ARGS is defined before referencing it. Defaults to empty
 # if not defined.
 export CMAKE_ARGS="${CMAKE_ARGS:-}"

diff --git a/setup.py b/setup.py
@@ -423,6 +423,11 @@ def run(self):
                 "devtools/bundled_program/schema/scalar_type.fbs",
                 "devtools/bundled_program/serialize/scalar_type.fbs",
             ),
+            # Install executorch-config.cmake to the root of the package.
+            (
+                "build/executorch-config.cmake",
+                "executorch-config.cmake",
+            ),
         ]
         # Copy all the necessary headers into include/executorch/ so that they can
         # be found in the pip package. This is the subset of headers that are