Disable more tests.

toitlang · Oct 10, 2024 · e8750bf · e8750bf
1 parent dcd9c77
commit e8750bf
Show file tree

Hide file tree

Showing 7 changed files with 82 additions and 28 deletions.
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
@@ -79,11 +79,21 @@ endforeach()
 
 include(fail.cmake OPTIONAL)
 
+set(OPTIMIZATION_IS_OVERRIDDEN FALSE)
+if (DEFINED ENV{TOIT_OPTIMIZATION_OVERRIDE})
+  set(OPTIMIZATION_IS_OVERRIDDEN TRUE)
+endif()
+
 foreach(file ${ALL_TESTS})
   if("${file}" IN_LIST TOIT_SKIP_TESTS)
     continue()
   endif()
 
+  if(OPTIMIZATION_IS_OVERRIDDEN AND "${file}" IN_LIST TOIT_OPTIMIZATION_SKIP_TESTS)
+    message("Skipping ${file} as optimization is overridden")
+    continue()
+  endif()
+
   set(TOIT_ARGS) # Arguments to toit.run.
   set(TEST_ARGS) # Arguments to the test.
 

diff --git a/tests/fail.cmake b/tests/fail.cmake
@@ -34,6 +34,11 @@ list(APPEND TOIT_FLAKY_TESTS
   tests/tls-resume-session-test.toit
 )
 
+list(APPEND TOIT_OPTIMIZATION_SKIP_TESTS
+  # The following tests only work with standard optimizations.
+  tests/bytes-allocated-test.toit
+)
+
 if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Windows" OR "${CMAKE_SYSTEM_NAME}" STREQUAL "MSYS")
   list(APPEND TOIT_FAILING_TESTS
     tests/time-test.toit # https://github.com/toitlang/toit/issues/1369

diff --git a/tests/negative/CMakeLists.txt b/tests/negative/CMakeLists.txt
@@ -24,13 +24,16 @@ set(NORMALIZE_GOLD ${CMAKE_CURRENT_SOURCE_DIR}/../tools/normalize_gold.cmake)
 include(fail.cmake)
 
 set(ASSERT_IS_ENABLED TRUE)
-if (ENV{TOIT_ASSERT_OVERRIDE} STREQUAL "0")
+if ($ENV{TOIT_ASSERT_OVERRIDE} STREQUAL "0")
   set(ASSERT_IS_ENABLED FALSE)
 endif()
+
+set(OPTIMIZATION_IS_OVERRIDDEN FALSE)
+if (DEFINED ENV{TOIT_OPTIMIZATION_OVERRIDE})
+  set(OPTIMIZATION_IS_OVERRIDDEN TRUE)
+endif()
+
 foreach(file ${TOIT_TESTS})
-  if ("${file}" IN_LIST ASSERT_TESTS AND NOT ASSERT_IS_ENABLED)
-    continue()
-  endif()
   get_filename_component(name ${file} NAME_WE)
   set(lib_dir "./lib")
   set(toit_file ${CMAKE_CURRENT_SOURCE_DIR}/${file})
@@ -40,32 +43,42 @@ foreach(file ${TOIT_TESTS})
   file(RELATIVE_PATH relative ${TOIT_SDK_SOURCE_DIR} ${toit_file})
   string(REPLACE " " "__" test_name ${relative})
 
-  if(NOT "${test_name}" IN_LIST TOIT_SKIP_TESTS)
-    set(TEST_EXPECTATION_NAME "${test_name}")
-    if("${test_name}" IN_LIST TOIT_FAILING_TESTS)
-      set(TEST_EXPECTATION_NAME "${test_name}-expected-to-fail")
-    endif()
+  if ("${file}" IN_LIST ASSERT_TESTS AND NOT ASSERT_IS_ENABLED)
+    message("Skipping ${file} as asserts are disabled")
+    continue()
+  endif()
+  if(OPTIMIZATION_IS_OVERRIDDEN AND "${test_name}" IN_LIST TOIT_OPTIMIZATION_SKIP_TESTS)
+    message("Skipping ${file} as optimization is overridden")
+    continue()
+  endif()
+  if ("${test_name}" IN_LIST TOIT_SKIP_TESTS)
+    continue()
+  endif()
+
+  set(TEST_EXPECTATION_NAME "${test_name}")
+  if("${test_name}" IN_LIST TOIT_FAILING_TESTS)
+    set(TEST_EXPECTATION_NAME "${test_name}-expected-to-fail")
+  endif()
 
-    add_test(
-      NAME ${TEST_EXPECTATION_NAME}
-      COMMAND ${CMAKE_COMMAND}
-          -DTOITVM=$<TARGET_FILE:toit.run>
-          "-DTEST=${relative}"
-          "-DGOLD=${gold_file}"
-          "-DLIB_DIR=${lib_dir}"
-          "-DNORMALIZE_GOLD=${NORMALIZE_GOLD}"
-          "-DTEST_ROOT=${TOIT_SDK_SOURCE_DIR}"
-          "-DGIT_VERSION=${TOIT_GIT_VERSION}"
-          "-DTMP=${CMAKE_BINARY_DIR}/tmp"
-          "-DCMAKE_SYSTEM_NAME=${CMAKE_SYSTEM_NAME}"
-          -P "${CMAKE_CURRENT_SOURCE_DIR}/run.cmake"
-      WORKING_DIRECTORY ${TOIT_SDK_SOURCE_DIR}
-    )
-    set_tests_properties(${TEST_EXPECTATION_NAME} PROPERTIES TIMEOUT 40)
+  add_test(
+    NAME ${TEST_EXPECTATION_NAME}
+    COMMAND ${CMAKE_COMMAND}
+        -DTOITVM=$<TARGET_FILE:toit.run>
+        "-DTEST=${relative}"
+        "-DGOLD=${gold_file}"
+        "-DLIB_DIR=${lib_dir}"
+        "-DNORMALIZE_GOLD=${NORMALIZE_GOLD}"
+        "-DTEST_ROOT=${TOIT_SDK_SOURCE_DIR}"
+        "-DGIT_VERSION=${TOIT_GIT_VERSION}"
+        "-DTMP=${CMAKE_BINARY_DIR}/tmp"
+        "-DCMAKE_SYSTEM_NAME=${CMAKE_SYSTEM_NAME}"
+        -P "${CMAKE_CURRENT_SOURCE_DIR}/run.cmake"
+    WORKING_DIRECTORY ${TOIT_SDK_SOURCE_DIR}
+  )
+  set_tests_properties(${TEST_EXPECTATION_NAME} PROPERTIES TIMEOUT 40)
 
-    if("${test_name}" IN_LIST TOIT_FAILING_TESTS)
-      set_tests_properties(${TEST_EXPECTATION_NAME} PROPERTIES WILL_FAIL TRUE)
-    endif()
+  if("${test_name}" IN_LIST TOIT_FAILING_TESTS)
+    set_tests_properties(${TEST_EXPECTATION_NAME} PROPERTIES WILL_FAIL TRUE)
   endif()
 
   file(RELATIVE_PATH relative_gold ${TOIT_SDK_SOURCE_DIR} ${gold_file})

diff --git a/tests/negative/fail.cmake b/tests/negative/fail.cmake
@@ -15,3 +15,8 @@
 
 set(TOIT_FAILING_TESTS
 )
+
+list(APPEND TOIT_OPTIMIZATION_SKIP_TESTS
+  # The following tests only work with standard optimizations.
+  tests/negative/field-type5-test.toit
+)
diff --git a/tests/profiler/CMakeLists.txt b/tests/profiler/CMakeLists.txt
@@ -17,6 +17,11 @@ file(GLOB PROFILER_TESTS "*-test.toit")
 
 include(fail.cmake)
 
+if (DEFINED ENV{TOIT_OPTIMIZATION_OVERRIDE})
+  # Don't run profiler tests if the optimization level is overridden.
+  set(PROFILER_TESTS)
+endif()
+
 foreach(profiler_test ${PROFILER_TESTS})
   get_filename_component(base ${profiler_test} NAME_WE)
   string(REGEX REPLACE "-test$" "" test_name ${base})

diff --git a/tests/toit/CMakeLists.txt b/tests/toit/CMakeLists.txt
@@ -14,15 +14,26 @@
 # directory of this repository.
 
 file(GLOB TOIT_BIN_TESTS "*-test.toit")
+file(GLOB ASSERT_TESTS "assert*-test.toit")
 
 include(fail.cmake OPTIONAL)
 
 set(TEST_SDK_DIR "${CMAKE_BINARY_DIR}/sdk")
 set(TOIT_BIN_SOURCE "${TOIT_SDK_SOURCE_DIR}/tools/toit.toit")
 
+set(ASSERT_IS_OVERRIDDEN FALSE)
+if (DEFINED ENV{TOIT_ASSERT_OVERRIDE})
+  set(ASSERT_IS_OVERRIDDEN TRUE)
+endif()
+
 foreach(file ${TOIT_BIN_TESTS})
   get_filename_component(base ${file} NAME_WE)
 
+  if (ASSERT_IS_OVERRIDDEN AND "${file}" IN_LIST ASSERT_TESTS)
+    message("Skipping ${file} as asserts are overridden")
+    continue()
+  endif()
+
   file(RELATIVE_PATH toit_bin_test_name ${TOIT_SDK_SOURCE_DIR} ${file})
   if (NOT "${toit_bin_test_name}" IN_LIST TOIT_SKIP_TESTS)
     set(TEST_EXPECTATION_NAME "${toit_bin_test_name}")

diff --git a/tests/toitp/CMakeLists.txt b/tests/toitp/CMakeLists.txt
@@ -63,6 +63,11 @@ add_test(
   DEPENDS ${BYTECODE_LIST}
   )
 
+if (DEFINED ENV{TOIT_OPTIMIZATION_OVERRIDE})
+  # Don't run toitp tests if the optimization level is overridden.
+  set(TOITP_TESTS)
+endif()
+
 foreach(file ${TOITP_TESTS})
   get_filename_component(base ${file} NAME_WE)
   string(REGEX REPLACE "-toitp-test$" "" test_name ${base})