Merge branch 'main' of https://github.com/oneapi-src/unified-runtime …

…into ianayl/2way-prefetch

CMakeLists.txt

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 cmake_minimum_required(VERSION 3.20.0 FATAL_ERROR)
-project(unified-runtime VERSION 0.11.0)
+project(unified-runtime VERSION 0.12.0)
 
 # Check if unified runtime is built as a standalone project.
 if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR OR UR_STANDALONE_BUILD)

include/ur_api.h

@@ -7,7 +7,7 @@
  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  * @file ur_api.h
- * @version v0.11-r0
+ * @version v0.12-r0
  *
  */
 #ifndef UR_API_H_INCLUDED
@@ -1156,7 +1156,8 @@ typedef enum ur_api_version_t {
     UR_API_VERSION_0_9 = UR_MAKE_VERSION(0, 9),      ///< version 0.9
     UR_API_VERSION_0_10 = UR_MAKE_VERSION(0, 10),    ///< version 0.10
     UR_API_VERSION_0_11 = UR_MAKE_VERSION(0, 11),    ///< version 0.11
-    UR_API_VERSION_CURRENT = UR_MAKE_VERSION(0, 11), ///< latest known version
+    UR_API_VERSION_0_12 = UR_MAKE_VERSION(0, 12),    ///< version 0.12
+    UR_API_VERSION_CURRENT = UR_MAKE_VERSION(0, 12), ///< latest known version
     /// @cond
     UR_API_VERSION_FORCE_UINT32 = 0x7fffffff
     /// @endcond

include/ur_api_funcs.def

@@ -8,7 +8,7 @@
  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  * @file ur_api_funcs.def
- * @version v0.11-r0
+ * @version v0.12-r0
  *
  */
 

include/ur_ddi.h

@@ -7,7 +7,7 @@
  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  * @file ur_ddi.h
- * @version v0.11-r0
+ * @version v0.12-r0
  *
  */
 #ifndef UR_DDI_H_INCLUDED

include/ur_print.hpp

@@ -7,7 +7,7 @@
  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  * @file ur_print.hpp
- * @version v0.11-r0
+ * @version v0.12-r0
  *
  */
 #ifndef UR_PRINT_HPP

scripts/Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = "Intel One API Unified Runtime API"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = v0.11
+PROJECT_NUMBER         = v0.12
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

scripts/core/platform.yml

@@ -148,6 +148,9 @@ etors:
     - name: "0_11"
       value: "$X_MAKE_VERSION( 0, 11 )"
       desc: "version 0.11"
+    - name: "0_12"
+      value: "$X_MAKE_VERSION( 0, 12 )"
+      desc: "version 0.12"
 --- #--------------------------------------------------------------------------
 type: function
 desc: "Returns the API version supported by the specified platform"

scripts/parse_specs.py

@@ -21,8 +21,10 @@
 from version import Version
 
 
-default_version = Version("0.11")
-all_versions = [Version(ver) for ver in ["0.6", "0.7", "0.8", "0.9", "0.10", "0.11"]]
+default_version = Version("0.12")
+all_versions = [
+    Version(ver) for ver in ["0.6", "0.7", "0.8", "0.9", "0.10", "0.11", "0.12"]
+]
 
 """
     preprocess object

source/adapters/level_zero/CMakeLists.txt

@@ -127,6 +127,7 @@ if(UR_BUILD_ADAPTER_L0_V2)
         ${CMAKE_CURRENT_SOURCE_DIR}/common.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/device.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/platform.hpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.hpp
@@ -135,10 +136,12 @@ if(UR_BUILD_ADAPTER_L0_V2)
         ${CMAKE_CURRENT_SOURCE_DIR}/device.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/ur_interface_loader.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/platform.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/program.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/usm_p2p.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/virtual_mem.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/tensor_map.cpp
         # v2-only sources

source/adapters/level_zero/adapter.cpp

@@ -9,6 +9,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "adapter.hpp"
+#include "common.hpp"
 #include "ur_level_zero.hpp"
 #include <iomanip>
 
@@ -162,7 +163,7 @@ ur_result_t initPlatforms(PlatformVec &platforms,
     ZE2UR_CALL(zeDriverGet, (&ZeDriverGetCount, ZeDriverGetHandles.data()));
   }
   if (ZeDriverGetCount == 0 && GlobalAdapter->ZeInitDriversCount == 0) {
-    logger::debug("\nNo Valid L0 Drivers found.\n");
+    logger::error("\nNo Valid L0 Drivers found.\n");
     return UR_RESULT_SUCCESS;
   }
 
@@ -376,7 +377,9 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
                     static_cast<int>(L0InitFlags));
       GlobalAdapter->ZeInitResult = ZE_CALL_NOCHECK(zeInit, (L0InitFlags));
       if (GlobalAdapter->ZeInitResult != ZE_RESULT_SUCCESS) {
-        logger::debug("\nzeInit failed with {}\n", GlobalAdapter->ZeInitResult);
+        const char *ErrorString = "Unknown";
+        zeParseError(GlobalAdapter->ZeInitResult, ErrorString);
+        logger::error("\nzeInit failed with {}\n", ErrorString);
       }
 
       bool useInitDrivers = false;
@@ -422,8 +425,9 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
           if (GlobalAdapter->ZeInitDriversResult == ZE_RESULT_SUCCESS) {
             GlobalAdapter->InitDriversSupported = true;
           } else {
-            logger::debug("\nzeInitDrivers failed with {}\n",
-                          GlobalAdapter->ZeInitDriversResult);
+            const char *ErrorString = "Unknown";
+            zeParseError(GlobalAdapter->ZeInitDriversResult, ErrorString);
+            logger::error("\nzeInitDrivers failed with {}\n", ErrorString);
           }
         }
       }
@@ -441,6 +445,7 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
 
     // Absorb the ZE_RESULT_ERROR_UNINITIALIZED and just return 0 Platforms.
     if (*GlobalAdapter->ZeResult == ZE_RESULT_ERROR_UNINITIALIZED) {
+      logger::error("Level Zero Uninitialized\n");
       result = std::move(platforms);
       return;
     }

source/adapters/level_zero/common.cpp

@@ -88,7 +88,7 @@ ZeUSMImportExtension ZeUSMImport;
 
 std::map<std::string, int> *ZeCallCount = nullptr;
 
-inline void zeParseError(ze_result_t ZeError, const char *&ErrorString) {
+void zeParseError(ze_result_t ZeError, const char *&ErrorString) {
   switch (ZeError) {
 #define ZE_ERRCASE(ERR)                                                        \
   case ERR:                                                                    \

source/adapters/level_zero/common.hpp

@@ -340,6 +340,9 @@ bool setEnvVar(const char *name, const char *value);
 // Map Level Zero runtime error code to UR error code.
 ur_result_t ze2urResult(ze_result_t ZeResult);
 
+// Parse Level Zero error code and return the error string.
+void zeParseError(ze_result_t ZeError, const char *&ErrorString);
+
 // Trace a call to Level-Zero RT
 #define ZE2UR_CALL(ZeName, ZeArgs)                                             \
   {                                                                            \

source/adapters/level_zero/device.cpp

@@ -654,9 +654,15 @@ ur_result_t urDeviceGetInfo(
     return ReturnValue(Device->ZeDeviceProperties->physicalEUSimdWidth / 4);
   case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_DOUBLE:
   case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_DOUBLE:
+    // Must return 0 for *vector_width_double* if the device does not have fp64.
+    if (!(Device->ZeDeviceModuleProperties->flags & ZE_DEVICE_MODULE_FLAG_FP64))
+      return ReturnValue(uint32_t{0});
     return ReturnValue(Device->ZeDeviceProperties->physicalEUSimdWidth / 8);
   case UR_DEVICE_INFO_NATIVE_VECTOR_WIDTH_HALF:
   case UR_DEVICE_INFO_PREFERRED_VECTOR_WIDTH_HALF:
+    // Must return 0 for *vector_width_half* if the device does not have fp16.
+    if (!(Device->ZeDeviceModuleProperties->flags & ZE_DEVICE_MODULE_FLAG_FP16))
+      return ReturnValue(uint32_t{0});
     return ReturnValue(Device->ZeDeviceProperties->physicalEUSimdWidth / 2);
   case UR_DEVICE_INFO_MAX_NUM_SUB_GROUPS: {
     // Max_num_sub_Groups = maxTotalGroupSize/min(set of subGroupSizes);

source/adapters/level_zero/event.cpp

@@ -145,6 +145,10 @@ ur_result_t urEnqueueEventsWait(
     std::unique_lock<ur_shared_mutex> Lock(Queue->Mutex);
     resetCommandLists(Queue);
   }
+  if (OutEvent && (*OutEvent)->Completed) {
+    UR_CALL(CleanupCompletedEvent((*OutEvent), false, false));
+    UR_CALL(urEventReleaseInternal((*OutEvent)));
+  }
 
   return UR_RESULT_SUCCESS;
 }
@@ -955,7 +959,6 @@ ur_result_t urEventCreateWithNativeHandle(
     UREvent = new ur_event_handle_t_(ZeEvent, nullptr /* ZeEventPool */,
                                      Context, UR_EXT_COMMAND_TYPE_USER,
                                      Properties->isNativeHandleOwned);
-
     UREvent->RefCountExternal++;
 
   } catch (const std::bad_alloc &) {

source/adapters/level_zero/image.cpp

@@ -1237,7 +1237,7 @@ ur_result_t urBindlessImagesImportExternalSemaphoreExp(
   }
 
   ZE2UR_CALL(UrPlatform->ZeExternalSemaphoreExt.zexImportExternalSemaphoreExp,
-             (hDevice->ZeDevice, &ExtSemaphoreHandle, &SemDesc));
+             (hDevice->ZeDevice, &SemDesc, &ExtSemaphoreHandle));
   *phExternalSemaphoreHandle =
       (ur_exp_external_semaphore_handle_t)ExtSemaphoreHandle;
 
@@ -1310,7 +1310,7 @@ ur_result_t urBindlessImagesWaitExternalSemaphoreExp(
       reinterpret_cast<ze_intel_external_semaphore_exp_handle_t>(hSemaphore);
   ZE2UR_CALL(UrPlatform->ZeExternalSemaphoreExt
                  .zexCommandListAppendWaitExternalSemaphoresExp,
-             (ZeCommandList, &hExtSemaphore, &WaitParams, 1, ZeEvent,
+             (ZeCommandList, 1, &hExtSemaphore, &WaitParams, ZeEvent,
               WaitList.Length, WaitList.ZeEventList));
 
   return UR_RESULT_SUCCESS;
@@ -1373,7 +1373,7 @@ ur_result_t urBindlessImagesSignalExternalSemaphoreExp(
 
   ZE2UR_CALL(UrPlatform->ZeExternalSemaphoreExt
                  .zexCommandListAppendSignalExternalSemaphoresExp,
-             (ZeCommandList, &hExtSemaphore, &SignalParams, 1, ZeEvent,
+             (ZeCommandList, 1, &hExtSemaphore, &SignalParams, ZeEvent,
               WaitList.Length, WaitList.ZeEventList));
 
   return UR_RESULT_SUCCESS;

source/adapters/level_zero/physical_mem.cpp

@@ -10,9 +10,13 @@
 
 #include "physical_mem.hpp"
 #include "common.hpp"
-#include "context.hpp"
 #include "device.hpp"
-#include "ur_level_zero.hpp"
+
+#ifdef UR_ADAPTER_LEVEL_ZERO_V2
+#include "v2/context.hpp"
+#else
+#include "context.hpp"
+#endif
 
 namespace ur::level_zero {
 
@@ -25,7 +29,7 @@ ur_result_t urPhysicalMemCreate(
   PhysicalMemDesc.size = size;
 
   ze_physical_mem_handle_t ZePhysicalMem;
-  ZE2UR_CALL(zePhysicalMemCreate, (hContext->ZeContext, hDevice->ZeDevice,
+  ZE2UR_CALL(zePhysicalMemCreate, (hContext->getZeHandle(), hDevice->ZeDevice,
                                    &PhysicalMemDesc, &ZePhysicalMem));
   try {
     *phPhysicalMem = new ur_physical_mem_handle_t_(ZePhysicalMem, hContext);
@@ -46,8 +50,8 @@ ur_result_t urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem) {
   if (!hPhysicalMem->RefCount.decrementAndTest())
     return UR_RESULT_SUCCESS;
 
-  ZE2UR_CALL(zePhysicalMemDestroy,
-             (hPhysicalMem->Context->ZeContext, hPhysicalMem->ZePhysicalMem));
+  ZE2UR_CALL(zePhysicalMemDestroy, (hPhysicalMem->Context->getZeHandle(),
+                                    hPhysicalMem->ZePhysicalMem));
   delete hPhysicalMem;
 
   return UR_RESULT_SUCCESS;

source/adapters/level_zero/platform.hpp

@@ -119,17 +119,17 @@ struct ur_platform_handle_t_ : public _ur_platform {
   struct ZeExternalSemaphoreExtension {
     bool Supported = false;
     ze_result_t (*zexImportExternalSemaphoreExp)(
-        ze_device_handle_t, ze_intel_external_semaphore_exp_handle_t *,
-        const ze_intel_external_semaphore_exp_desc_t *);
+        ze_device_handle_t, const ze_intel_external_semaphore_exp_desc_t *,
+        ze_intel_external_semaphore_exp_handle_t *);
     ze_result_t (*zexCommandListAppendWaitExternalSemaphoresExp)(
-        ze_command_list_handle_t,
+        ze_command_list_handle_t, unsigned int,
         const ze_intel_external_semaphore_exp_handle_t *,
-        const ze_intel_external_semaphore_wait_exp_params_t *, unsigned int,
+        const ze_intel_external_semaphore_wait_exp_params_t *,
         ze_event_handle_t, uint32_t, ze_event_handle_t *);
     ze_result_t (*zexCommandListAppendSignalExternalSemaphoresExp)(
-        ze_command_list_handle_t,
+        ze_command_list_handle_t, size_t,
         const ze_intel_external_semaphore_exp_handle_t *,
-        const ze_intel_external_semaphore_signal_exp_params_t *, size_t,
+        const ze_intel_external_semaphore_signal_exp_params_t *,
         ze_event_handle_t, uint32_t, ze_event_handle_t *);
     ze_result_t (*zexDeviceReleaseExternalSemaphoreExp)(
         ze_intel_external_semaphore_exp_handle_t);

source/adapters/level_zero/v2/api.cpp

@@ -88,81 +88,6 @@ ur_result_t urSamplerCreateWithNativeHandle(
   return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
 }
 
-ur_result_t urVirtualMemGranularityGetInfo(
-    ur_context_handle_t hContext, ur_device_handle_t hDevice,
-    ur_virtual_mem_granularity_info_t propName, size_t propSize,
-    void *pPropValue, size_t *pPropSizeRet) {
-  logger::error("{} function not implemented!", __FUNCTION__);
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
-}
-
-ur_result_t urVirtualMemReserve(ur_context_handle_t hContext,
-                                const void *pStart, size_t size,
-                                void **ppStart) {
-  logger::error("{} function not implemented!", __FUNCTION__);
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
-}
-
-ur_result_t urVirtualMemFree(ur_context_handle_t hContext, const void *pStart,
-                             size_t size) {
-  logger::error("{} function not implemented!", __FUNCTION__);
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
-}
-
-ur_result_t urVirtualMemMap(ur_context_handle_t hContext, const void *pStart,
-                            size_t size, ur_physical_mem_handle_t hPhysicalMem,
-                            size_t offset,
-                            ur_virtual_mem_access_flags_t flags) {
-  logger::error("{} function not implemented!", __FUNCTION__);
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
-}
-
-ur_result_t urVirtualMemUnmap(ur_context_handle_t hContext, const void *pStart,
-                              size_t size) {
-  logger::error("{} function not implemented!", __FUNCTION__);
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
-}
-
-ur_result_t urVirtualMemSetAccess(ur_context_handle_t hContext,
-                                  const void *pStart, size_t size,
-                                  ur_virtual_mem_access_flags_t flags) {
-  logger::error("{} function not implemented!", __FUNCTION__);
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
-}
-
-ur_result_t urVirtualMemGetInfo(ur_context_handle_t hContext,
-                                const void *pStart, size_t size,
-                                ur_virtual_mem_info_t propName, size_t propSize,
-                                void *pPropValue, size_t *pPropSizeRet) {
-  logger::error("{} function not implemented!", __FUNCTION__);
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
-}
-
-ur_result_t urPhysicalMemCreate(ur_context_handle_t hContext,
-                                ur_device_handle_t hDevice, size_t size,
-                                const ur_physical_mem_properties_t *pProperties,
-                                ur_physical_mem_handle_t *phPhysicalMem) {
-  logger::error("{} function not implemented!", __FUNCTION__);
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
-}
-
-ur_result_t urPhysicalMemRetain(ur_physical_mem_handle_t hPhysicalMem) {
-  logger::error("{} function not implemented!", __FUNCTION__);
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
-}
-
-ur_result_t urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem) {
-  logger::error("{} function not implemented!", __FUNCTION__);
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
-}
-
-UR_APIEXPORT ur_result_t UR_APICALL urPhysicalMemGetInfo(
-    ur_physical_mem_handle_t hPhysicalMem, ur_physical_mem_info_t propName,
-    size_t propSize, void *pPropValue, size_t *pPropSizeRet) {
-  logger::error("{} function not implemented!", __FUNCTION__);
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
-}
-
 ur_result_t
 urKernelSetArgSampler(ur_kernel_handle_t hKernel, uint32_t argIndex,
                       const ur_kernel_arg_sampler_properties_t *pProperties,