Add CTS macro for marking tests as known failures in code

The new `UUR_KNOWN_FAILURE_ON` macro can be used to skip tests on devices where the test is known to fail. This can be done for a devices in an adapter or by substring match of the device name. For example: ```cpp UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"}); ``` This invocation is also used in a few places in order to have clean CTS runs on machines with this iGPU.
oneapi-src · Nov 29, 2024 · 28cfbbb · 28cfbbb
1 parent 1c59bea
commit 28cfbbb
Show file tree

Hide file tree

Showing 6 changed files with 99 additions and 3 deletions.
diff --git a/test/conformance/kernel/urKernelSetArgValue.cpp b/test/conformance/kernel/urKernelSetArgValue.cpp
@@ -3,6 +3,7 @@
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+#include "uur/known_failure.h"
 #include <uur/fixtures.h>
 
 struct urKernelSetArgValueTest : uur::urKernelTest {
@@ -45,6 +46,7 @@ TEST_P(urKernelSetArgValueTest, InvalidKernelArgumentIndex) {
 }
 
 TEST_P(urKernelSetArgValueTest, InvalidKernelArgumentSize) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
     ASSERT_EQ_RESULT(UR_RESULT_ERROR_INVALID_KERNEL_ARGUMENT_SIZE,
                      urKernelSetArgValue(kernel, 2, 0, nullptr, &arg_value));
 }
diff --git a/test/conformance/memory/urMemImageCreate.cpp b/test/conformance/memory/urMemImageCreate.cpp
@@ -2,6 +2,7 @@
 // Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#include "uur/known_failure.h"
 #include <uur/fixtures.h>
 #include <uur/raii.h>
 
@@ -188,6 +189,7 @@ TEST_P(urMemImageCreateTest, InvalidNullPointerImageFormat) {
 }
 
 TEST_P(urMemImageCreateTest, InvalidSize) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
 
     uur::raii::Mem image_handle = nullptr;
 

diff --git a/test/conformance/memory/urMemImageCreateWithImageFormatParam.cpp b/test/conformance/memory/urMemImageCreateWithImageFormatParam.cpp
@@ -2,6 +2,7 @@
 // Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#include "uur/known_failure.h"
 #include <uur/fixtures.h>
 #include <vector>
 
@@ -90,6 +91,7 @@ UUR_TEST_SUITE_P(
     uur::deviceTestWithParamPrinter<ur_image_format_t>);
 
 TEST_P(urMemImageCreateTestWithImageFormatParam, Success) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
     ur_image_channel_order_t channel_order =
         std::get<1>(GetParam()).channelOrder;
     ur_image_channel_type_t channel_type = std::get<1>(GetParam()).channelType;

diff --git a/test/conformance/program/urProgramGetFunctionPointer.cpp b/test/conformance/program/urProgramGetFunctionPointer.cpp
@@ -3,6 +3,7 @@
 // See LICENSE.TXT
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+#include "uur/known_failure.h"
 #include <uur/fixtures.h>
 
 struct urProgramGetFunctionPointerTest : uur::urProgramTest {
@@ -20,6 +21,7 @@ struct urProgramGetFunctionPointerTest : uur::urProgramTest {
 UUR_INSTANTIATE_KERNEL_TEST_SUITE_P(urProgramGetFunctionPointerTest);
 
 TEST_P(urProgramGetFunctionPointerTest, Success) {
+    UUR_KNOWN_FAILURE_ON(uur::OpenCL{"Intel(R) UHD Graphics 770"});
     void *function_pointer = nullptr;
     ASSERT_SUCCESS(urProgramGetFunctionPointer(
         device, program, function_name.data(), &function_pointer));

diff --git a/test/conformance/testing/include/uur/known_failure.h b/test/conformance/testing/include/uur/known_failure.h
@@ -0,0 +1,88 @@
+// Copyright (C) 2024 Intel Corporation
+// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
+// See LICENSE.TXT
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef UR_CONFORMANCE_INCLUDE_KNOWN_FAILURE_H_INCLUDED
+#define UR_CONFORMANCE_INCLUDE_KNOWN_FAILURE_H_INCLUDED
+
+#include "uur/utils.h"
+#include <string>
+#include <vector>
+
+namespace uur {
+struct DeviceMatcher {
+    DeviceMatcher(ur_platform_backend_t backend,
+                  std::vector<std::string> deviceNames)
+        : backend(backend), deviceNames(deviceNames) {}
+
+    ur_platform_backend_t backend;
+    std::vector<std::string> deviceNames;
+};
+
+struct OpenCL : DeviceMatcher {
+    OpenCL(std::initializer_list<std::string> il)
+        : DeviceMatcher(UR_PLATFORM_BACKEND_OPENCL, {il.begin(), il.end()}) {}
+};
+
+struct LevelZero : DeviceMatcher {
+    LevelZero(std::initializer_list<std::string> il)
+        : DeviceMatcher(UR_PLATFORM_BACKEND_LEVEL_ZERO,
+                        {il.begin(), il.end()}) {}
+};
+
+struct CUDA : DeviceMatcher {
+    CUDA(std::initializer_list<std::string> il)
+        : DeviceMatcher(UR_PLATFORM_BACKEND_CUDA, {il.begin(), il.end()}) {}
+};
+
+struct HIP : DeviceMatcher {
+    HIP(std::initializer_list<std::string> il)
+        : DeviceMatcher(UR_PLATFORM_BACKEND_HIP, {il.begin(), il.end()}) {}
+};
+
+struct NativeCPU : DeviceMatcher {
+    NativeCPU(std::initializer_list<std::string> il)
+        : DeviceMatcher(UR_PLATFORM_BACKEND_NATIVE_CPU,
+                        {il.begin(), il.end()}) {}
+};
+
+inline bool isKnownFailureOn(ur_platform_handle_t platform,
+                             ur_device_handle_t device,
+                             std::vector<DeviceMatcher> deviceMatchers) {
+    for (const auto &deviceMatcher : deviceMatchers) {
+        ur_platform_backend_t backend = UR_PLATFORM_BACKEND_UNKNOWN;
+        uur::GetPlatformInfo<ur_platform_backend_t>(
+            platform, UR_PLATFORM_INFO_BACKEND, backend);
+        if (deviceMatcher.backend != backend) {
+            continue;
+        }
+        if (deviceMatcher.deviceNames.empty()) {
+            return true;
+        }
+        std::string deviceName;
+        uur::GetDeviceInfo<std::string>(device, UR_DEVICE_INFO_NAME,
+                                        deviceName);
+        for (const auto &unsupportedDeviceName : deviceMatcher.deviceNames) {
+            if (deviceName.find(unsupportedDeviceName) != std::string::npos) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+} // namespace uur
+
+#define UUR_KNOWN_FAILURE_ON(...)                                              \
+    if (uur::isKnownFailureOn(platform, device, {__VA_ARGS__})) {              \
+        std::string platformName;                                              \
+        uur::GetPlatformInfo<std::string>(platform, UR_PLATFORM_INFO_NAME,     \
+                                          platformName);                       \
+        std::string deviceName;                                                \
+        uur::GetDeviceInfo<std::string>(device, UR_DEVICE_INFO_NAME,           \
+                                        deviceName);                           \
+        GTEST_SKIP() << "Known failure on: " << platformName << ", "           \
+                     << deviceName;                                            \
+    }
+
+#endif // UR_CONFORMANCE_INCLUDE_KNOWN_FAILURE_H_INCLUDED
diff --git a/test/conformance/testing/include/uur/utils.h b/test/conformance/testing/include/uur/utils.h
@@ -43,12 +43,12 @@ ur_result_t GetInfo(ObjectTy object, InfoTy info, Callable cb, T &out_value) {
 
     // special case for strings
     if constexpr (std::is_same_v<std::string, T>) {
-        std::vector<char> data(size);
-        result = cb(object, info, size, data.data(), nullptr);
+        std::string value(size, '\0');
+        result = cb(object, info, size, value.data(), nullptr);
         if (result != UR_RESULT_SUCCESS) {
             return result;
         }
-        out_value = std::string(data.data(), data.size());
+        out_value = value.substr(0, value.find_last_of('\0'));
         return UR_RESULT_SUCCESS;
     } else {
         if (size != sizeof(T)) {