diff --git a/.typos.toml b/.typos.toml index f42228417c..56f0ad44e0 100644 --- a/.typos.toml +++ b/.typos.toml @@ -24,3 +24,4 @@ USCALED = "USCALED" Datas = "Datas" HSA = "HSA" VALU = "VALU" +Derivate = "Derivate" diff --git a/CMakeLists.txt b/CMakeLists.txt index 79dfc302b2..483dbccfc9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,22 +33,21 @@ if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) set(LLPC_IS_STANDALONE ON) endif() +### Version info ### +include(cmake/llpc_version.cmake) +add_llpc_version_projects() + ### Top-level VKGC Interface ### add_library(vkgc INTERFACE) ### VKGC header-only library ### add_library(vkgc_headers INTERFACE) +target_link_libraries(vkgc_headers INTERFACE llpc_version) + ### Cached Project Options ############################################################################################# option(LLPC_BUILD_TOOLS "LLPC build all tools" OFF) - ### Options that affect the headers #################################################################################### -if (LLPC_CLIENT_INTERFACE_MAJOR_VERSION) - target_compile_definitions(vkgc_headers INTERFACE LLPC_CLIENT_INTERFACE_MAJOR_VERSION=${LLPC_CLIENT_INTERFACE_MAJOR_VERSION}) -else() - message(FATAL_ERROR "Client of LLPC must set LLPC_CLIENT_INTERFACE_MAJOR_VERSION") -endif() - #if VKI_BUILD_GFX11 if(LLPC_BUILD_GFX11) target_compile_definitions(vkgc_headers INTERFACE VKI_BUILD_GFX11) diff --git a/cmake/continuations.cmake b/cmake/continuations.cmake index 8500b5fb17..cf7b96ed80 100644 --- a/cmake/continuations.cmake +++ b/cmake/continuations.cmake @@ -25,12 +25,14 @@ set(LLPC_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/..") +include("${LLPC_SOURCE_DIR}/cmake/llpc_version.cmake") include("${LLPC_SOURCE_DIR}/cmake/compilerutils.cmake") # Macro to add continuations and its dependencies as LLVM external projects. # This appends the project names to LLVM_EXTERNAL_PROJECTS and sets each LLVM_EXTERNAL_*_SOURCE_DIR, # all in the caller's scope. macro(add_continuations_projects) + add_llpc_version_projects() add_compilerutils_projects() if (NOT continuations IN_LIST LLVM_EXTERNAL_PROJECTS) if (NOT llvm_dialects IN_LIST LLVM_EXTERNAL_PROJECTS) diff --git a/cmake/lgc.cmake b/cmake/lgc.cmake index a07c04481f..87aae4a8bb 100644 --- a/cmake/lgc.cmake +++ b/cmake/lgc.cmake @@ -25,12 +25,14 @@ set(LLPC_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/..") +include("${LLPC_SOURCE_DIR}/cmake/llpc_version.cmake") include("${LLPC_SOURCE_DIR}/cmake/continuations.cmake") # Macro to add LGC and its dependencies as LLVM external projects. # This appends the project names to LLVM_EXTERNAL_PROJECTS and sets each LLVM_EXTERNAL_*_SOURCE_DIR, # all in the caller's scope. macro(add_lgc_projects) + add_llpc_version_projects() add_continuations_projects() if (NOT lgc IN_LIST LLVM_EXTERNAL_PROJECTS) if (NOT llvm_dialects IN_LIST LLVM_EXTERNAL_PROJECTS) diff --git a/cmake/llpc_version.cmake b/cmake/llpc_version.cmake new file mode 100644 index 0000000000..881eb464a6 --- /dev/null +++ b/cmake/llpc_version.cmake @@ -0,0 +1,34 @@ +## + ####################################################################################################################### + # + # Copyright (c) 2023 Advanced Micro Devices, Inc. All Rights Reserved. + # + # Permission is hereby granted, free of charge, to any person obtaining a copy + # of this software and associated documentation files (the "Software"), to deal + # in the Software without restriction, including without limitation the rights + # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + # copies of the Software, and to permit persons to whom the Software is + # furnished to do so, subject to the following conditions: + # + # The above copyright notice and this permission notice shall be included in all + # copies or substantial portions of the Software. + # + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + # SOFTWARE. + # + ####################################################################################################################### + +set(LLPC_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/..") + +macro(add_llpc_version_projects) + if (NOT TARGET llpc_version) + # Force the binary directory to account for the possibility that LLPC is + # taken from an external source directory. + add_subdirectory(${LLPC_SOURCE_DIR}/version ${CMAKE_CURRENT_BINARY_DIR}/llpc_version) + endif() +endmacro() diff --git a/compilerutils/CMakeLists.txt b/compilerutils/CMakeLists.txt index 1d97dd27c3..588e02e7f8 100644 --- a/compilerutils/CMakeLists.txt +++ b/compilerutils/CMakeLists.txt @@ -22,6 +22,7 @@ add_llvm_library(LLVMCompilerUtils Analysis Core Support + TransformUtils ) target_include_directories(LLVMCompilerUtils PUBLIC @@ -35,3 +36,6 @@ set_compiler_options(LLVMCompilerUtils) target_compile_features(LLVMCompilerUtils PUBLIC cxx_std_17) set_target_properties(LLVMCompilerUtils PROPERTIES CXX_EXTENSIONS OFF) + +add_subdirectory(tool/cross-module-inline) +add_subdirectory(test) diff --git a/compilerutils/include/compilerutils/CompilerUtils.h b/compilerutils/include/compilerutils/CompilerUtils.h index 458324de52..0ca8d72b81 100644 --- a/compilerutils/include/compilerutils/CompilerUtils.h +++ b/compilerutils/include/compilerutils/CompilerUtils.h @@ -38,15 +38,6 @@ #include "llvm/IR/Attributes.h" #include "llvm/IR/IRBuilder.h" -namespace llvm { - -class CallInst; -class Function; -class Type; -class Value; - -} // namespace llvm - namespace CompilerUtils { // Create an LLVM function call to the named function. The callee is built @@ -66,6 +57,56 @@ llvm::CallInst *createNamedCall(llvm::IRBuilder<> &, llvm::StringRef, llvm::Type llvm::Function *mutateFunctionArguments(llvm::Function &, llvm::Type *, const llvm::ArrayRef, llvm::AttributeList); +// Create a new function based on another function, copying attributes and +// other properties. +// Specify targetModule to create the function in a different module than f. +llvm::Function *cloneFunctionHeader(llvm::Function &f, llvm::FunctionType *newType, llvm::AttributeList attributes, + llvm::Module *targetModule = nullptr); + +// Overload of cloneFunctionHeader that takes the new attributes for arguments and preserves the rest. +llvm::Function *cloneFunctionHeader(llvm::Function &f, llvm::FunctionType *newType, + llvm::ArrayRef argAttrs, llvm::Module *targetModule = nullptr); + +struct CrossModuleInlinerResult { + llvm::Value *returnValue; + llvm::iterator_range newBBs; +}; + +// The class caches already mapped constants. Reusing an instance of this class is more efficient than creating a new +// instance every time but it does not have an impact on the generated code. +// One CrossModuleInliner instance must only be used for a single target module, otherwise things can go wrong. +class CrossModuleInliner { +public: + CrossModuleInliner() = default; + + // Inline a call to a function even if the called function is in a different module. + // If the result of that function call should be used, a use must exist before calling this function. + // Returns the new created basic blocks. These blocks may also contain instructions that were already + // there before, if the function got inlined into an existing block. + // + // The insertion point of existing IRBuilders may have their insertion point invalidated because this + // function splits basic blocks. + // They can be made functional again with b.SetInsertPoint(&*b.GetInsertPoint()). + llvm::iterator_range inlineCall(llvm::CallBase &cb); + + // Inline a call to a function even if the called function is in a different module. + // Returns the result of the call and new created basic blocks. These blocks may also contain instructions that were + // already there before, if the function got inlined into an existing block. + // + // This is a convenience wrapper around inlineCall(CallBase&). As users of the callee's return value are not known + // while inlining, using this function can result in slightly less folding in the IR. + CrossModuleInlinerResult inlineCall(llvm::IRBuilder<> &b, llvm::Function *callee, + llvm::ArrayRef args = std::nullopt); + + // Find a global value (function or variable) that was copied by the cross-module inliner. + // Arguments are the global from the source module and the target module. Returns the corresponding global from the + // target module. + llvm::GlobalValue *findCopiedGlobal(llvm::GlobalValue &sourceGv, llvm::Module &targetModule); + +private: + llvm::SmallDenseMap mappedGlobals; +}; + } // namespace CompilerUtils #endif diff --git a/compilerutils/include/compilerutils/LoweringPointerTupleMap.h b/compilerutils/include/compilerutils/LoweringPointerTupleMap.h new file mode 100644 index 0000000000..8111ac7157 --- /dev/null +++ b/compilerutils/include/compilerutils/LoweringPointerTupleMap.h @@ -0,0 +1,241 @@ +/* + *********************************************************************************************************************** + * + * Copyright (c) 2020-2023 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + **********************************************************************************************************************/ +/** + *********************************************************************************************************************** + * @file LoweringPointerTupleMap.h + * @brief A key-value map from keys to tuples of pointers that is optimized for value and type lowering uses + *********************************************************************************************************************** + */ + +#pragma once + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include +#include + +namespace compilerutils { + +/// @brief A key-value map from pointer keys to tuples of pointers that is optimized for value and type lowering uses +/// +/// This map is optimized using two main assumptions: +/// +/// 1. The vast majority of keys are mapped to a single pointer. +/// 2. Keys can be updated, but this happens rarely, and there is no need to reclaim memory except when the map as a +/// whole is destroyed. +/// +/// Pointers and references into the map are @em not stable. +/// +/// The map does not distinguish between missing entries and entries mapped to an empty tuple. +/// +/// The map can optionally track all places in which a value appears, for an efficient implementation of +/// @ref replaceAllUsesOfWith. +template class LoweringPointerTupleMap { + template struct AlignOfPointee; + + template struct AlignOfPointee { static constexpr size_t value = alignof(PointeeT); }; + + static_assert(std::is_pointer_v); + static_assert(AlignOfPointee::value >= 2); + + static_assert(std::is_pointer_v); + static_assert(AlignOfPointee::value >= 2); + + using Map = llvm::DenseMap; + + // Tracking the reverse mapping takes up some space which we only use if it was requested. + struct Empty {}; + using ReverseMap = std::conditional_t>, Empty>; + + /// If requested, track the locations in which each value is mentioned. + [[no_unique_address]] ReverseMap m_reverseMap; + + /// Map keys to values. + /// + /// For 1-1 mappings, this stores a value pointer. + /// For 1-N mappings, this stores ((index << 1) | 1), where index is the index + /// into m_extra. + Map m_map; + + /// (size, values...) sequences stored contiguously for every 1-N mapping. + std::vector m_extra; + + llvm::ArrayRef decode(const uintptr_t *encoded) const { + if ((*encoded & 1) == 0) + return llvm::ArrayRef(reinterpret_cast(encoded), 1); + + size_t index = *encoded >> 1; + assert(index < m_extra.size()); + size_t count = m_extra[index]; + assert(count >= 2); + assert(count < m_extra.size() - index); + const ValueT *begin = reinterpret_cast(&m_extra[index + 1]); + return llvm::ArrayRef(begin, begin + count); + } + +public: + bool empty() const { return m_map.empty(); } + size_t size() const { return m_map.size(); } + + void clear() { + m_map.clear(); + m_extra.clear(); + + if constexpr (TrackReverse) + m_reverseMap.clear(); + } + + llvm::ArrayRef lookup(KeyT key) const { + auto it = m_map.find(key); + if (it == m_map.end()) + return {}; + + return decode(&it->second); + } + + llvm::ArrayRef set(KeyT key, llvm::ArrayRef value) { + if (value.empty()) { + auto it = m_map.find(key); + if (it == m_map.end()) + return {}; + + if constexpr (TrackReverse) + this->clearReverseMap(it); + m_map.erase(it); + return {}; + } + + auto [it, inserted] = m_map.try_emplace(key); + if constexpr (TrackReverse) { + if (!inserted) + this->clearReverseMap(it); + } + + uintptr_t encoded; + + if (value.size() == 1) { + encoded = reinterpret_cast(value[0]); + assert((encoded & 1) == 0); + + if constexpr (TrackReverse) + m_reverseMap[value[0]].push_back(reinterpret_cast(key)); + } else { + size_t index = m_extra.size(); + + m_extra.push_back(value.size()); + for (size_t i = 0; i < value.size(); ++i) { + // Use a memcpy to store the value because even though m_extra is a + // vector of uintptr_t, we will access it via ValueT pointers later. + m_extra.emplace_back(); + memcpy(&m_extra.back(), &value[i], sizeof(uintptr_t)); + if constexpr (TrackReverse) + m_reverseMap[value[i]].push_back(((index + 1 + i) << 1) | 1); + } + + encoded = (index << 1) | 1; + assert((encoded >> 1) == index); + } + + it->second = encoded; + + return decode(&it->second); + } + + // Replace a value that may have previously been recorded as part of a mapping + // with another value. + // + // If the values in the map are `llvm::Value *`s, this can be used if RAUW is + // performed on IR, as in: + // @code + // toReplace->replaceAllUsesWith(with); + // map.replaceAllUsesWith(toReplace, with); + // @endcode + // + // Note that this method is only available when the TrackReverse template + // parameter is set to true. + // + // @param toReplace : the mapping value to be replaced + // @param with : the new value to replace it with in all mappings in which + // it appears + void replaceAllUsesOfWith(ValueT toReplace, ValueT with) { + static_assert(TrackReverse); + + if (toReplace == with) + return; + + auto toReplaceIt = m_reverseMap.find(toReplace); + if (toReplaceIt == m_reverseMap.end()) + return; + + llvm::SmallVector occurrences = std::move(toReplaceIt->second); + m_reverseMap.erase(toReplaceIt); + + for (uintptr_t occurrence : occurrences) { + if (occurrence & 1) { + // Use a memcpy to store the value because even though m_extra is a + // vector of uintptr_t, we will access it via ValueT pointers later. + memcpy(&m_extra[occurrence >> 1], &with, sizeof(uintptr_t)); + } else { + m_map.find(reinterpret_cast(occurrence))->second = reinterpret_cast(with); + } + } + + auto withIt = m_reverseMap.find(with); + if (withIt != m_reverseMap.end()) { + withIt->second.append(occurrences); + } else { + m_reverseMap.try_emplace(with, std::move(occurrences)); + } + } + +private: + // Clear the reverse map for all entries in mapIt. + void clearReverseMap(typename Map::iterator mapIt) { + auto clearEntry = [&](ValueT oldValue, uintptr_t occurrence) { + auto it = m_reverseMap.find(oldValue); + assert(it != m_reverseMap.end()); + if (it->second.size() == 1) { + assert(it->second[0] == occurrence); + m_reverseMap.erase(it); + } else { + *llvm::find(it->second, occurrence) = it->second.back(); + it->second.pop_back(); + } + }; + + if ((mapIt->second & 1) == 0) { + ValueT oldValue = reinterpret_cast(mapIt->second); + clearEntry(oldValue, reinterpret_cast(mapIt->first)); + } else { + size_t index = mapIt->second >> 1; + size_t count = m_extra[index]; + for (size_t i = 0; i != count; ++i) + clearEntry(reinterpret_cast(m_extra[index + 1 + i]), ((index + 1 + i) << 1) | 1); + } + } +}; + +} // namespace compilerutils diff --git a/compilerutils/include/compilerutils/TypeLowering.h b/compilerutils/include/compilerutils/TypeLowering.h index 5bec7d1e96..fb57d825da 100644 --- a/compilerutils/include/compilerutils/TypeLowering.h +++ b/compilerutils/include/compilerutils/TypeLowering.h @@ -53,6 +53,7 @@ #pragma once +#include "compilerutils/LoweringPointerTupleMap.h" #include "llvm-dialects/Dialect/Visitor.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" @@ -166,27 +167,13 @@ class TypeLowering { llvm::SmallVector> m_rules; llvm::SmallVector> m_constantRules; - /// Cache 1-1 mappings of types (including no-op mappings). - llvm::DenseMap m_unaryTypeConversions; - - /// Cache 1-N mappings of types. - llvm::DenseMap> m_multiTypeConversions; + /// Cache mappings of types (including no-op mappings). + compilerutils::LoweringPointerTupleMap m_typeConversions; llvm::IRBuilder<> m_builder; /// Map original values to type-converted values. - /// - /// For 1-1 mappings, this stores a value pointer. - /// For 1-N mappings, this stores ((index << 1) | 1), where index is the index - /// into m_convertedValueList at which the converted values can be found. - llvm::DenseMap m_valueMap; - std::vector m_convertedValueList; - - /// Reverse map of values that occur as type-converted values to where they - /// occur. The vector elements are either a value pointer (for 1-1 mapped - /// values) or ((index << 1) | 1), where index is the index into - /// m_convertedValueList. - llvm::DenseMap> m_valueReverseMap; + compilerutils::LoweringPointerTupleMap m_valueMap; std::vector>> m_phis; std::vector m_instructionsToErase; diff --git a/compilerutils/lib/CompilerUtils.cpp b/compilerutils/lib/CompilerUtils.cpp index 8d3f0ab589..3fa8892098 100644 --- a/compilerutils/lib/CompilerUtils.cpp +++ b/compilerutils/lib/CompilerUtils.cpp @@ -24,11 +24,19 @@ **********************************************************************************************************************/ #include "compilerutils/CompilerUtils.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/IR/Attributes.h" #include "llvm/IR/Function.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Transforms/Utils/Cloning.h" +#include "llvm/Transforms/Utils/ValueMapper.h" + +#define DEBUG_TYPE "compilerutils" using namespace llvm; @@ -89,12 +97,226 @@ CallInst *CompilerUtils::createNamedCall(IRBuilder<> &builder, StringRef funcNam Function *CompilerUtils::mutateFunctionArguments(Function &fn, Type *retTy, const ArrayRef argTys, AttributeList attributes) { FunctionType *newFnTy = FunctionType::get(retTy, argTys, false); - auto *newFn = Function::Create(newFnTy, fn.getLinkage()); - newFn->copyAttributesFrom(&fn); - newFn->copyMetadata(&fn, 0); + auto *newFn = cloneFunctionHeader(fn, newFnTy, attributes); newFn->takeName(&fn); - newFn->setAttributes(attributes); newFn->splice(newFn->begin(), &fn); - fn.getParent()->getFunctionList().insertAfter(fn.getIterator(), newFn); return newFn; } + +Function *CompilerUtils::cloneFunctionHeader(Function &f, FunctionType *newType, AttributeList attributes, + Module *targetModule) { + LLVM_DEBUG(dbgs() << "Cloning function " << f.getName() << " with new type " << *newType << "\n"); + Function *newFunc = Function::Create(newType, f.getLinkage(), "", targetModule); + + if (targetModule) { + assert(targetModule != f.getParent() && "targetModule cannot be the same as the current module"); + newFunc->setName(f.getName()); + } else { + // Insert new function before f to facilitate writing tests + f.getParent()->getFunctionList().insert(f.getIterator(), newFunc); + } + + newFunc->copyAttributesFrom(&f); + newFunc->setSubprogram(f.getSubprogram()); + newFunc->setAttributes(attributes); + newFunc->copyMetadata(&f, 0); + return newFunc; +} + +Function *CompilerUtils::cloneFunctionHeader(Function &f, FunctionType *newType, ArrayRef argAttrs, + Module *targetModule) { + const AttributeList fAttrs = f.getAttributes(); + const AttributeList attributes = + AttributeList::get(f.getContext(), fAttrs.getFnAttrs(), fAttrs.getRetAttrs(), argAttrs); + return cloneFunctionHeader(f, newType, attributes, targetModule); +} + +namespace { + +// Get the name of a global that is copied to a different module for inlining. +std::string getCrossModuleName(GlobalValue &gv) { + return (Twine(gv.getName()) + ".cloned." + gv.getParent()->getName()).str(); +} + +class CrossModuleValueMaterializer : public ValueMaterializer { +public: + CrossModuleValueMaterializer(Module *targetMod, CompilerUtils::CrossModuleInliner &inliner, + SmallDenseMap &mapped) + : targetMod(targetMod), inliner(&inliner), mapped(&mapped) {} + virtual ~CrossModuleValueMaterializer() = default; + + void setMapper(ValueMapper *mapper) { this->mapper = mapper; } + + virtual Value *materialize(Value *v) override { + if (auto *gv = dyn_cast(v)) { + if (gv->getParent() == targetMod) + return nullptr; + + auto *newGv = moveGlobalValueToNewModule(gv); + return newGv; + } + return nullptr; + } + +private: + GlobalValue *moveGlobalValueToNewModule(GlobalValue *gv) { + if (auto *existing = inliner->findCopiedGlobal(*gv, *targetMod)) + return existing; + + auto newName = getCrossModuleName(*gv); + if (auto *callee = dyn_cast(gv)) { + if (!callee->isDeclaration()) { + report_fatal_error( + Twine("Cross module inlining does not support functions with calls to functions with a body. " + "Run the inliner before trying to inline across modules (trying to call '") + + callee->getName() + "')"); + } + + // Create a function declaration + auto *newGv = + CompilerUtils::cloneFunctionHeader(*callee, callee->getFunctionType(), callee->getAttributes(), targetMod); + newGv->setName(newName); + + (*mapped)[gv] = newGv; + return newGv; + } + + if (auto *gVar = dyn_cast(gv)) { + // Create a global with the correct type + auto *newGv = new GlobalVariable(*targetMod, gVar->getValueType(), gVar->isConstant(), gVar->getLinkage(), + nullptr, newName, nullptr, gVar->getThreadLocalMode(), gVar->getAddressSpace()); + newGv->copyAttributesFrom(gVar); + if (gVar->hasInitializer()) { + // Recursively map initializer + auto *newInit = mapper->mapConstant(*gVar->getInitializer()); + newGv->setInitializer(newInit); + } + + (*mapped)[gv] = newGv; + return newGv; + } + + report_fatal_error("Encountered unknown global object while inlining"); + } + + Module *targetMod; + CompilerUtils::CrossModuleInliner *inliner; + SmallDenseMap *mapped; + ValueMapper *mapper; +}; + +} // anonymous namespace + +iterator_range CompilerUtils::CrossModuleInliner::inlineCall(CallBase &cb) { + auto *calleeFunc = cb.getCalledFunction(); + assert(calleeFunc && "Cannot find called function"); + LLVM_DEBUG(dbgs() << "Inlining '" << calleeFunc->getName() << "' across modules\n"); + + Function *targetFunc = cb.getFunction(); + auto *targetMod = targetFunc->getParent(); + auto callBb = cb.getParent()->getIterator(); + auto callBbSuccessor = callBb; + ++callBbSuccessor; + const bool callBbHasSuccessor = callBbSuccessor != targetFunc->end(); + const size_t bbCount = targetFunc->size(); + // Save uses of the return value + SmallVector users(cb.users()); + + const bool hasInstBefore = cb.getIterator() != callBb->begin(); + auto instBefore = hasInstBefore ? --cb.getIterator() : cb.getIterator(); + auto instAfter = ++cb.getIterator(); + const bool hasInstAfter = instAfter != callBb->end(); + + if (calleeFunc->isDeclaration()) { + // We cannot inline declarations, but declarations are useful for testing + // purposes, so we allow declarations if we are not cross-module. + assert(calleeFunc->getParent() == targetMod && "Cannot inline declarations cross-module"); + return make_range(callBb, callBb); + } + + // Copy code + InlineFunctionInfo ifi; + auto res = InlineFunction(cb, ifi); + if (!res.isSuccess()) + report_fatal_error(Twine("Failed to inline ") + calleeFunc->getName() + ": " + res.getFailureReason()); + + // The first block of the inlined function gets concatenated into the calling BB, + // the rest of the inlined function follows the calling BB. + auto firstNewBb = callBb; + auto lastNewBb = callBbHasSuccessor ? callBbSuccessor : targetFunc->end(); + + // Check that the number of basic blocks is as expected + assert(bbCount + static_cast(std::distance(firstNewBb, lastNewBb)) - 1 == targetFunc->size() && + "Did not find all inlined blocks"); + + if (calleeFunc->getParent() != targetMod) { + // The name is important because it is used for copying and re-finding globals + assert(!calleeFunc->getParent()->getName().empty() && "Can only inline from modules that have a name"); + + // Look for references to global values and replace them with global values in the new module + CrossModuleValueMaterializer materializer{targetMod, *this, mappedGlobals}; + ValueToValueMapTy map; + ValueMapper mapper{map, RF_IgnoreMissingLocals, nullptr, &materializer}; + materializer.setMapper(&mapper); + for (auto bb = firstNewBb; bb != lastNewBb; bb++) { + bool skipBeforeInsts = hasInstBefore && bb == firstNewBb; + for (auto &i : *bb) { + // Skip instructions before and after the original call + if (skipBeforeInsts) { + if (&i == &*instBefore) + skipBeforeInsts = false; + continue; + } + if (hasInstAfter && &i == &*instAfter) + break; + + mapper.remapInstruction(i); + } + assert((bb != firstNewBb || !hasInstBefore || !skipBeforeInsts) && "Did not find first instruction"); + } + + // If the inlined function returned a constant, that gets inlined into the users of the original value. Iterate over + // these to catch all global values + for (auto *u : users) + mapper.remapInstruction(*cast(u)); + } + + return make_range(firstNewBb, lastNewBb); +} + +CompilerUtils::CrossModuleInlinerResult +CompilerUtils::CrossModuleInliner::inlineCall(IRBuilder<> &b, llvm::Function *callee, + llvm::ArrayRef args) { + auto *call = b.CreateCall(callee, args); + // Create a fake use, so we can get the result of the inlined function. + FreezeInst *fakeUse = nullptr; + if (!callee->getReturnType()->isVoidTy()) + fakeUse = cast(b.CreateFreeze(call)); + + auto newBBs = inlineCall(*call); + + Value *result = nullptr; + if (fakeUse) { + result = fakeUse->getOperand(0); + fakeUse->eraseFromParent(); + } + b.SetInsertPoint(&*b.GetInsertPoint()); + return {result, newBBs}; +} + +GlobalValue *CompilerUtils::CrossModuleInliner::findCopiedGlobal(GlobalValue &sourceGv, Module &targetModule) { + assert(sourceGv.getParent() != &targetModule && "This function only finds copies across modules"); + assert(sourceGv.hasName() && "Cannot find a global value that does not have a name"); + + if (auto found = mappedGlobals.find(&sourceGv); found != mappedGlobals.end()) { + assert(found->second->getParent() == &targetModule && + "The CrossModuleInliner can only be used with a single target module"); + return found->second; + } + + auto *gv = + targetModule.getNamedValue((Twine(sourceGv.getName()) + ".cloned." + sourceGv.getParent()->getName()).str()); + if (gv) + assert(gv->getValueType() == sourceGv.getValueType()); + return gv; +} diff --git a/compilerutils/lib/TypeLowering.cpp b/compilerutils/lib/TypeLowering.cpp index 4beadfc032..c864391f8d 100644 --- a/compilerutils/lib/TypeLowering.cpp +++ b/compilerutils/lib/TypeLowering.cpp @@ -191,23 +191,16 @@ void TypeLowering::addConstantRule(std::function rule) { // // @param ty : the type ArrayRef TypeLowering::convertType(Type *ty) { - auto unaryIt = m_unaryTypeConversions.find(ty); - if (unaryIt != m_unaryTypeConversions.end()) - return ArrayRef(unaryIt->second); - - auto multiIt = m_multiTypeConversions.find(ty); - if (multiIt != m_multiTypeConversions.end()) - return multiIt->second; + auto conversion = m_typeConversions.lookup(ty); + if (!conversion.empty()) + return conversion; for (const auto &rule : reverse(m_rules)) { SmallVector types = rule(*this, ty); if (types.empty()) continue; - if (types.size() == 1) - return ArrayRef(m_unaryTypeConversions.try_emplace(ty, types[0]).first->second); - - return m_multiTypeConversions.try_emplace(ty, std::move(types)).first->second; + return m_typeConversions.set(ty, types); } llvm_unreachable("core/fallback rule should prevent us from reaching this point"); @@ -252,8 +245,8 @@ SmallVector TypeLowering::getValue(Value *val) { // // @param val : the value SmallVector TypeLowering::getValueOptional(Value *val) { - auto valueIt = m_valueMap.find(val); - if (valueIt == m_valueMap.end()) { + auto mapping = m_valueMap.lookup(val); + if (mapping.empty()) { auto *constant = dyn_cast(val); if (!constant) return {}; @@ -278,19 +271,10 @@ SmallVector TypeLowering::getValueOptional(Value *val) { recordValue(val, converted); - valueIt = m_valueMap.find(val); - assert(valueIt != m_valueMap.end()); - } - - if ((valueIt->second & 1) == 0) { - return SmallVector(ArrayRef(reinterpret_cast(valueIt->second))); + mapping = m_valueMap.lookup(val); } - size_t begin = valueIt->second >> 1; - auto typeIt = m_multiTypeConversions.find(val->getType()); - assert(typeIt != m_multiTypeConversions.end()); - size_t count = typeIt->second.size(); - return SmallVector(ArrayRef(&m_convertedValueList[begin], count)); + return SmallVector(mapping); } // ===================================================================================================================== @@ -317,29 +301,12 @@ void TypeLowering::replaceInstruction(Instruction *inst, ArrayRef mappi // @param val : the value for which a mapping is recorded // @param mapping : the mapping that is recorded for the value void TypeLowering::recordValue(Value *val, ArrayRef mapping) { - assert(!m_valueMap.count(val)); - - if (mapping.size() == 1) { - m_valueMap.try_emplace(val, reinterpret_cast(mapping[0])); -#ifndef NDEBUG - auto types = convertType(val->getType()); - assert(types.size() == 1); - assert(types[0] == mapping[0]->getType()); -#endif - m_valueReverseMap[mapping[0]].emplace_back(reinterpret_cast(val)); - return; - } + assert(m_valueMap.lookup(val).empty()); - uintptr_t index = m_convertedValueList.size(); - uintptr_t code = (index << 1) | 1; - m_convertedValueList.insert(m_convertedValueList.end(), mapping.begin(), mapping.end()); - m_valueMap.try_emplace(val, code); - for (auto e : llvm::enumerate(mapping)) { - m_valueReverseMap[e.value()].emplace_back(((index + e.index()) << 1) | 1); - } + (void)m_valueMap.set(val, mapping); - // Unconditionally perform the conversion to ensure that it is available in - // getValue. + // Unconditionally perform the type conversion to ensure that it is available + // in getValue. auto types = convertType(val->getType()); assert(types.size() == mapping.size()); for (size_t idx = 0; idx < types.size(); ++idx) { @@ -370,30 +337,7 @@ void TypeLowering::eraseInstruction(llvm::Instruction *inst) { // @param with : the new value to replace it with in all mappings in which // it appears void TypeLowering::replaceMappingWith(Value *toReplace, Value *with) { - if (toReplace == with) - return; - - auto toReplaceIt = m_valueReverseMap.find(toReplace); - if (toReplaceIt == m_valueReverseMap.end()) - return; - - SmallVector occurrences = std::move(toReplaceIt->second); - m_valueReverseMap.erase(toReplaceIt); - - for (uintptr_t occurrence : occurrences) { - if (occurrence & 1) { - m_convertedValueList[occurrence >> 1] = with; - } else { - m_valueMap.find(reinterpret_cast(occurrence))->second = reinterpret_cast(with); - } - } - - auto withIt = m_valueReverseMap.find(with); - if (withIt != m_valueReverseMap.end()) { - withIt->second.append(occurrences); - } else { - m_valueReverseMap.try_emplace(with, std::move(occurrences)); - } + m_valueMap.replaceAllUsesOfWith(toReplace, with); } // ===================================================================================================================== @@ -472,8 +416,6 @@ bool TypeLowering::finishCleanup() { m_functionsToErase.clear(); m_valueMap.clear(); - m_convertedValueList.clear(); - m_valueReverseMap.clear(); return changed; } diff --git a/compilerutils/test/CMakeLists.txt b/compilerutils/test/CMakeLists.txt new file mode 100644 index 0000000000..4de3e114be --- /dev/null +++ b/compilerutils/test/CMakeLists.txt @@ -0,0 +1,27 @@ +set(COMPILERUTILS_TEST_DEPENDS cross-module-inline FileCheck count not) +add_custom_target(compilerutils-test-depends DEPENDS ${COMPILERUTILS_TEST_DEPENDS}) +set_target_properties(compilerutils-test-depends PROPERTIES FOLDER "Tests") + +# required by lit.site.cfg.py.in +set(COMPILERUTILS_TEST_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + +# required by configure_lit_site_cfg +set(LLVM_LIT_OUTPUT_DIR ${LLVM_TOOLS_BINARY_DIR}) +configure_lit_site_cfg( + ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in + ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py + MAIN_CONFIG + ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py +) + +add_lit_testsuite(check-compilerutils "Running the compilerutils regression tests" + ${CMAKE_CURRENT_BINARY_DIR} + ${exclude_from_check_all} + DEPENDS ${COMPILERUTILS_TEST_DEPENDS} +) +set_target_properties(check-compilerutils PROPERTIES FOLDER "Tests") + +add_lit_testsuites(COMPILERUTILS ${CMAKE_CURRENT_SOURCE_DIR} + ${exclude_from_check_all} + DEPENDS ${COMPILERUTILS_TEST_DEPENDS} +) diff --git a/compilerutils/test/inc/link-constant-expr-global.ll b/compilerutils/test/inc/link-constant-expr-global.ll new file mode 100644 index 0000000000..57f2d5ddcd --- /dev/null +++ b/compilerutils/test/inc/link-constant-expr-global.ll @@ -0,0 +1,8 @@ +%struct.MyStruct = type { i32, i32 } + +@glob = external global %struct.MyStruct + +define i32 @inline_fun() { + %result = load i32, ptr getelementptr inbounds (%struct.MyStruct, ptr @glob, i64 0, i32 1) + ret i32 %result +} diff --git a/compilerutils/test/inc/link-control-flow.ll b/compilerutils/test/inc/link-control-flow.ll new file mode 100644 index 0000000000..9ebe35befe --- /dev/null +++ b/compilerutils/test/inc/link-control-flow.ll @@ -0,0 +1,15 @@ +@glob = global ptr @glob.1 +@glob.1 = global i32 5 + +define i32 @inline_fun(i32 %arg) { + %result = add i32 %arg, 1 + %cond = icmp uge i32 %result, 5 + br i1 %cond, label %ret, label %ret2 + +ret: + ret i32 %result + +ret2: + %res2 = ptrtoint ptr @glob to i32 + ret i32 %res2 +} diff --git a/compilerutils/test/inc/link-existing-func-name.ll b/compilerutils/test/inc/link-existing-func-name.ll new file mode 100644 index 0000000000..909b642d6a --- /dev/null +++ b/compilerutils/test/inc/link-existing-func-name.ll @@ -0,0 +1,7 @@ +; A different main function +declare i32 @main(i32) + +define i32 @inline_fun(i32 %arg) { + %result = call i32 @main(i32 %arg) + ret i32 %result +} diff --git a/compilerutils/test/inc/link-existing-global-name.ll b/compilerutils/test/inc/link-existing-global-name.ll new file mode 100644 index 0000000000..c3ec71f0de --- /dev/null +++ b/compilerutils/test/inc/link-existing-global-name.ll @@ -0,0 +1,6 @@ +@glob = external global i32 + +define i32 @inline_fun() { + %result = load i32, ptr @glob + ret i32 %result +} diff --git a/compilerutils/test/inc/link-fold-const.ll b/compilerutils/test/inc/link-fold-const.ll new file mode 100644 index 0000000000..7b35370023 --- /dev/null +++ b/compilerutils/test/inc/link-fold-const.ll @@ -0,0 +1,7 @@ +@glob = global ptr @glob.1 +@glob.1 = global i32 5 + +define i32 @inline_fun() { + %result = ptrtoint ptr @glob to i32 + ret i32 %result +} diff --git a/compilerutils/test/inc/link-func-metadata.ll b/compilerutils/test/inc/link-func-metadata.ll new file mode 100644 index 0000000000..40d13d0f79 --- /dev/null +++ b/compilerutils/test/inc/link-func-metadata.ll @@ -0,0 +1,8 @@ +declare !md !0 i32 @main(i32) + +define i32 @inline_fun(i32 %arg) { + %result = call i32 @main(i32 %arg) + ret i32 %result +} + +!0 = !{!"abc"} diff --git a/compilerutils/test/inc/link-global-initializer.ll b/compilerutils/test/inc/link-global-initializer.ll new file mode 100644 index 0000000000..0e07ff47d0 --- /dev/null +++ b/compilerutils/test/inc/link-global-initializer.ll @@ -0,0 +1,7 @@ +@glob = external global i32 +@glob.1 = global ptr @glob + +define i32 @inline_fun() { + %result = ptrtoint ptr @glob.1 to i32 + ret i32 %result +} diff --git a/compilerutils/test/inc/link-global-same-struct.ll b/compilerutils/test/inc/link-global-same-struct.ll new file mode 100644 index 0000000000..ad6c9baa2c --- /dev/null +++ b/compilerutils/test/inc/link-global-same-struct.ll @@ -0,0 +1,9 @@ +%struct.MyStruct = type { i32, i32 } + +@glob = external global %struct.MyStruct + +define i32 @inline_fun() { + %ptr = getelementptr %struct.MyStruct, ptr @glob, i32 0, i32 1 + %result = load i32, ptr %ptr + ret i32 %result +} diff --git a/compilerutils/test/inc/link-global-same-type.ll b/compilerutils/test/inc/link-global-same-type.ll new file mode 100644 index 0000000000..c3ec71f0de --- /dev/null +++ b/compilerutils/test/inc/link-global-same-type.ll @@ -0,0 +1,6 @@ +@glob = external global i32 + +define i32 @inline_fun() { + %result = load i32, ptr @glob + ret i32 %result +} diff --git a/compilerutils/test/inc/link-simple.ll b/compilerutils/test/inc/link-simple.ll new file mode 100644 index 0000000000..26821e83c8 --- /dev/null +++ b/compilerutils/test/inc/link-simple.ll @@ -0,0 +1,4 @@ +define i32 @inline_fun(i32 %arg) { + %result = add i32 %arg, 1 + ret i32 %result +} diff --git a/compilerutils/test/link-constant-expr-global.ll b/compilerutils/test/link-constant-expr-global.ll new file mode 100644 index 0000000000..033db2108b --- /dev/null +++ b/compilerutils/test/link-constant-expr-global.ll @@ -0,0 +1,17 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool cross-module-inline --version 3 +; RUN: cross-module-inline %s %S/inc/link-constant-expr-global.ll --link inline_fun | FileCheck %s +; +; Access a global behind a constant expression. Check that linking creates a second global. + +@glob = external global float + +declare i32 @inline_fun() + +define i32 @main() { +; CHECK-LABEL: define i32 @main() { +; CHECK-NEXT: [[RESULT_I:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_MYSTRUCT:%.*]], ptr @"glob.cloned.{{.*}}", i64 0, i32 1), align 4 +; CHECK-NEXT: ret i32 [[RESULT_I]] +; + %result = call i32 @inline_fun() + ret i32 %result +} diff --git a/compilerutils/test/link-control-flow.ll b/compilerutils/test/link-control-flow.ll new file mode 100644 index 0000000000..b0175979ce --- /dev/null +++ b/compilerutils/test/link-control-flow.ll @@ -0,0 +1,24 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool cross-module-inline --version 3 +; RUN: cross-module-inline %s %S/inc/link-control-flow.ll --link inline_fun | FileCheck %s +; +; Inline a function with control-flow + +declare i32 @inline_fun(i32) + +define i32 @main(i32 %i) { +; CHECK-LABEL: define i32 @main( +; CHECK-SAME: i32 [[I:%.*]]) { +; CHECK-NEXT: [[RESULT_I:%.*]] = add i32 [[I]], 1 +; CHECK-NEXT: [[COND_I:%.*]] = icmp uge i32 [[RESULT_I]], 5 +; CHECK-NEXT: br i1 [[COND_I]], label [[RET_I:%.*]], label [[RET2_I:%.*]] +; CHECK: ret.i: +; CHECK-NEXT: br label [[INLINE_FUN_EXIT:%.*]] +; CHECK: ret2.i: +; CHECK-NEXT: br label [[INLINE_FUN_EXIT]] +; CHECK: inline_fun.exit: +; CHECK-NEXT: [[RESULT1:%.*]] = phi i32 [ [[RESULT_I]], [[RET_I]] ], [ ptrtoint (ptr @"glob.cloned.{{.*}}" to i32), [[RET2_I]] ] +; CHECK-NEXT: ret i32 [[RESULT1]] +; + %result = call i32 @inline_fun(i32 %i) + ret i32 %result +} diff --git a/compilerutils/test/link-existing-func-name.ll b/compilerutils/test/link-existing-func-name.ll new file mode 100644 index 0000000000..408db08bc6 --- /dev/null +++ b/compilerutils/test/link-existing-func-name.ll @@ -0,0 +1,15 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool cross-module-inline --version 3 +; RUN: cross-module-inline %s %S/inc/link-existing-func-name.ll --link inline_fun | FileCheck %s +; +; Linking creates a function declaration with a name that already exists in this module + +declare i32 @inline_fun(i32) + +define i32 @main() { +; CHECK-LABEL: define i32 @main() { +; CHECK-NEXT: [[RESULT_I:%.*]] = call i32 @"main.cloned.{{.*}}"(i32 4) +; CHECK-NEXT: ret i32 [[RESULT_I]] +; + %result = call i32 @inline_fun(i32 4) + ret i32 %result +} diff --git a/compilerutils/test/link-existing-global-multi.ll b/compilerutils/test/link-existing-global-multi.ll new file mode 100644 index 0000000000..f4bb6ff8de --- /dev/null +++ b/compilerutils/test/link-existing-global-multi.ll @@ -0,0 +1,26 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool cross-module-inline --version 3 +; RUN: cross-module-inline %s %S/inc/link-existing-global-name.ll --link inline_fun | FileCheck %s +; +; Link multiple times and check that the global is copied over only once, even if it is renamed. + +@glob = external global float + +declare i32 @inline_fun() + +define i32 @main() { +; CHECK-LABEL: define i32 @main() { +; CHECK-NEXT: [[RESULT_I2:%.*]] = load i32, ptr @"glob.cloned.{{.*}}", align 4 +; CHECK-NEXT: [[RESULT_I1:%.*]] = load i32, ptr @"glob.cloned.{{.*}}", align 4 +; CHECK-NEXT: [[RESULT_I:%.*]] = load i32, ptr @"glob.cloned.{{.*}}", align 4 +; CHECK-NEXT: [[A:%.*]] = add i32 [[RESULT_I2]], [[RESULT_I1]] +; CHECK-NEXT: [[B:%.*]] = add i32 [[A]], [[RESULT_I]] +; CHECK-NEXT: ret i32 [[B]] +; + %result = call i32 @inline_fun() + %result2 = call i32 @inline_fun() + %result3 = call i32 @inline_fun() + + %a = add i32 %result, %result2 + %b = add i32 %a, %result3 + ret i32 %b +} diff --git a/compilerutils/test/link-existing-global-name.ll b/compilerutils/test/link-existing-global-name.ll new file mode 100644 index 0000000000..9c2fe0a77a --- /dev/null +++ b/compilerutils/test/link-existing-global-name.ll @@ -0,0 +1,17 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool cross-module-inline --version 3 +; RUN: cross-module-inline %s %S/inc/link-existing-global-name.ll --link inline_fun | FileCheck %s +; +; Have global with the same name but different types in both modules. Check that linking creates a second global. + +@glob = external global float + +declare i32 @inline_fun() + +define i32 @main() { +; CHECK-LABEL: define i32 @main() { +; CHECK-NEXT: [[RESULT_I:%.*]] = load i32, ptr @"glob.cloned.{{.*}}", align 4 +; CHECK-NEXT: ret i32 [[RESULT_I]] +; + %result = call i32 @inline_fun() + ret i32 %result +} diff --git a/compilerutils/test/link-fold-const.ll b/compilerutils/test/link-fold-const.ll new file mode 100644 index 0000000000..0e34fea2fa --- /dev/null +++ b/compilerutils/test/link-fold-const.ll @@ -0,0 +1,22 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool cross-module-inline --version 3 +; RUN: cross-module-inline %s %S/inc/link-fold-const.ll --link inline_fun | FileCheck %s +; +; Check that copying globals still works if they are folded into later instructions. + +@glob = external global float + +declare i32 @inline_fun() + +define i32 @main() { +; CHECK-LABEL: define i32 @main() { +; CHECK-NEXT: br label [[RET:%.*]] +; CHECK: ret: +; CHECK-NEXT: ret i32 ptrtoint (ptr @"glob.cloned.{{.*}}" to i32) +; + %result = call i32 @inline_fun() + br label %ret + +; ret instruction is in a different basic block +ret: + ret i32 %result +} diff --git a/compilerutils/test/link-func-metadata.ll b/compilerutils/test/link-func-metadata.ll new file mode 100644 index 0000000000..da4930b3f6 --- /dev/null +++ b/compilerutils/test/link-func-metadata.ll @@ -0,0 +1,13 @@ +; RUN: cross-module-inline %s %S/inc/link-func-metadata.ll --link inline_fun | FileCheck %s +; +; Check that metadata of referenced functions is preserved + +; CHECK: declare !md !0 i32 @"main.cloned.{{.*}}"(i32) +; CHECK: !0 = !{!"abc"} + +declare i32 @inline_fun(i32) + +define i32 @main() { + %result = call i32 @inline_fun(i32 4) + ret i32 %result +} diff --git a/compilerutils/test/link-global-initializer.ll b/compilerutils/test/link-global-initializer.ll new file mode 100644 index 0000000000..e63b6c1b91 --- /dev/null +++ b/compilerutils/test/link-global-initializer.ll @@ -0,0 +1,19 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool cross-module-inline --version 3 +; RUN: cross-module-inline %s %S/inc/link-global-initializer.ll --link inline_fun | FileCheck %s +; +; Check that global initializers are copied + +; CHECK: @"glob.1.cloned.{{.*}}" = global ptr @"glob.cloned.{{.*}}" +; CHECK: @"glob.cloned.{{.*}}" = external global i32 + +@glob = external global float + +declare i32 @inline_fun() + +define i32 @main() { +; CHECK-LABEL: define i32 @main() { +; CHECK-NEXT: ret i32 ptrtoint (ptr @"glob.1.cloned.{{.*}}" to i32) +; + %result = call i32 @inline_fun() + ret i32 %result +} diff --git a/compilerutils/test/link-global-same-struct.ll b/compilerutils/test/link-global-same-struct.ll new file mode 100644 index 0000000000..9150ec4d58 --- /dev/null +++ b/compilerutils/test/link-global-same-struct.ll @@ -0,0 +1,19 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool cross-module-inline --version 3 +; RUN: cross-module-inline %s %S/inc/link-global-same-struct.ll --link inline_fun | FileCheck %s +; +; Have a global with an equal struct in both modules. Check that linking creates a second global. + +%struct.MyStruct = type { i32, i32 } + +@glob = external global %struct.MyStruct + +declare i32 @inline_fun() + +define i32 @main() { +; CHECK-LABEL: define i32 @main() { +; CHECK-NEXT: [[RESULT_I:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_MYSTRUCT_0:%.*]], ptr @"glob.cloned.{{.*}}", i64 0, i32 1), align 4 +; CHECK-NEXT: ret i32 [[RESULT_I]] +; + %result = call i32 @inline_fun() + ret i32 %result +} diff --git a/compilerutils/test/link-global-same-type.ll b/compilerutils/test/link-global-same-type.ll new file mode 100644 index 0000000000..358ddc6234 --- /dev/null +++ b/compilerutils/test/link-global-same-type.ll @@ -0,0 +1,17 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool cross-module-inline --version 3 +; RUN: cross-module-inline %s %S/inc/link-global-same-type.ll --link inline_fun | FileCheck %s +; +; Have global with the same name and same type in both modules. Check that linking creates a second global. + +@glob = external global i32 + +declare i32 @inline_fun() + +define i32 @main() { +; CHECK-LABEL: define i32 @main() { +; CHECK-NEXT: [[RESULT_I:%.*]] = load i32, ptr @"glob.cloned.{{.*}}", align 4 +; CHECK-NEXT: ret i32 [[RESULT_I]] +; + %result = call i32 @inline_fun() + ret i32 %result +} diff --git a/compilerutils/test/link-simple.ll b/compilerutils/test/link-simple.ll new file mode 100644 index 0000000000..b03d077515 --- /dev/null +++ b/compilerutils/test/link-simple.ll @@ -0,0 +1,14 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool cross-module-inline --version 3 +; RUN: cross-module-inline %s %S/inc/link-simple.ll --link inline_fun | FileCheck %s +; +; Simply inline a function + +declare i32 @inline_fun(i32) + +define i32 @main() { +; CHECK-LABEL: define i32 @main() { +; CHECK-NEXT: ret i32 5 +; + %result = call i32 @inline_fun(i32 4) + ret i32 %result +} diff --git a/compilerutils/test/lit.cfg.py b/compilerutils/test/lit.cfg.py new file mode 100644 index 0000000000..93dad026a2 --- /dev/null +++ b/compilerutils/test/lit.cfg.py @@ -0,0 +1,42 @@ +# -*- Python -*- + +# Configuration file for the 'lit' test runner. + +import os +import sys +import re +import platform +import subprocess + +import lit.util +import lit.formats +from lit.llvm import llvm_config +from lit.llvm.subst import FindTool +from lit.llvm.subst import ToolSubst + +# name: The name of this test suite. +config.name = 'COMPILERUTILS' + +# testFormat: The test format to use to interpret tests. +config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell) + +# suffixes: A list of file extensions to treat as test files. This is overridden +# by individual lit.local.cfg files in the test subdirectories. +config.suffixes = ['.ll'] + +# excludes: A list of directories to exclude from the testsuite. +config.excludes = ['inc'] + +# test_source_root: The root path where tests are located. +config.test_source_root = os.path.dirname(__file__) + +# test_exec_root: The root path where tests should be run. +config.test_exec_root = os.path.join(config.llvm_obj_root, 'test') + +if config.llvm_assertions in ['ON', '1']: + config.available_features.add('assertions') + +# Tweak the PATH to include the tools dir. +llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True) + +llvm_config.use_default_substitutions() diff --git a/compilerutils/test/lit.site.cfg.py.in b/compilerutils/test/lit.site.cfg.py.in new file mode 100644 index 0000000000..078eabe519 --- /dev/null +++ b/compilerutils/test/lit.site.cfg.py.in @@ -0,0 +1,27 @@ +@LIT_SITE_CFG_IN_HEADER@ + +import sys + +config.llvm_src_root = "@LLVM_SOURCE_DIR@" +config.llvm_obj_root = "@LLVM_BINARY_DIR@" +config.llvm_tools_dir = "@LLVM_TOOLS_DIR@" +config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@" +config.python_executable = "@PYTHON_EXECUTABLE@" + +# Support substitution of the tools_dir with user parameters. This is +# used when we can't determine the tool dir at configuration time. +try: + config.llvm_tools_dir = config.llvm_tools_dir % lit_config.params +except KeyError: + e = sys.exc_info()[1] + key, = e.args + lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key)) + +# Propagate CMake options used in lit feature tests. +config.llvm_assertions = "@LLVM_ENABLE_ASSERTIONS@" + +import lit.llvm +lit.llvm.initialize(lit_config, config) + +# Let the main config do the real work. +lit_config.load_config(config, "@COMPILERUTILS_TEST_SOURCE_DIR@/lit.cfg.py") diff --git a/compilerutils/tool/cross-module-inline/CMakeLists.txt b/compilerutils/tool/cross-module-inline/CMakeLists.txt new file mode 100644 index 0000000000..d77a1cbcc9 --- /dev/null +++ b/compilerutils/tool/cross-module-inline/CMakeLists.txt @@ -0,0 +1,40 @@ +## + ####################################################################################################################### + # + # Copyright (c) 2023 Advanced Micro Devices, Inc. All Rights Reserved. + # + # Permission is hereby granted, free of charge, to any person obtaining a copy + # of this software and associated documentation files (the "Software"), to deal + # in the Software without restriction, including without limitation the rights + # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + # copies of the Software, and to permit persons to whom the Software is + # furnished to do so, subject to the following conditions: + # + # The above copyright notice and this permission notice shall be included in all + # copies or substantial portions of the Software. + # + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + # SOFTWARE. + # + ####################################################################################################################### + +### LGC disassembler tool ############################################################################################ +set(LLVM_LINK_COMPONENTS + Support +) + +add_llvm_tool(cross-module-inline + cross-module-inline.cpp +) + +# others are linked in separately to account for both static and dynamic library +# builds. +llvm_map_components_to_libnames(extra_llvm_libs CompilerUtils) +target_link_libraries(cross-module-inline PRIVATE ${extra_llvm_libs}) + +set_compiler_options(cross-module-inline OFF) diff --git a/compilerutils/tool/cross-module-inline/cross-module-inline.cpp b/compilerutils/tool/cross-module-inline/cross-module-inline.cpp new file mode 100644 index 0000000000..a90c3ac83d --- /dev/null +++ b/compilerutils/tool/cross-module-inline/cross-module-inline.cpp @@ -0,0 +1,140 @@ +/* + *********************************************************************************************************************** + * + * Copyright (c) 2021-2023 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + **********************************************************************************************************************/ +/** + *********************************************************************************************************************** + * @file cross-module-inline.cpp + * @brief Command-line utility that allows to test cross-module inlining. + *********************************************************************************************************************** + */ + +#include "compilerutils/CompilerUtils.h" +#include "llvm/IRReader/IRReader.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/SourceMgr.h" +#include + +using namespace llvm; + +namespace { +// Input file for the module that is inlined to ("-" for stdin +cl::opt MainModule(cl::Positional, cl::ValueRequired, cl::desc("main_module")); +// Input file for the module that is inlined from ("-" for stdin) +cl::opt LinkModule(cl::Positional, cl::ValueRequired, cl::desc("link_module")); + +cl::list + LinkFunction("link", cl::desc("Name of the function to link and inline from the link_module to the main_module")); + +cl::opt OutFileName("o", cl::desc("Output filename ('-' for stdout)"), cl::value_desc("filename")); + +std::unique_ptr parseIr(LLVMContext &context, std::string &filename) { + llvm::SMDiagnostic error; + + llvm::ErrorOr> inputFileOrErr = + llvm::MemoryBuffer::getFileOrSTDIN(filename, /*IsText=*/false); + if (std::error_code errorCode = inputFileOrErr.getError()) { + auto error = SMDiagnostic(filename, SourceMgr::DK_Error, + "Could not open input file '" + filename + "': " + errorCode.message()); + error.print("cross-module-inline", errs()); + errs() << "\n"; + exit(EXIT_FAILURE); + } + auto inputFileBuffer = std::move(inputFileOrErr.get()); + + // Parse as IR file + auto mod = llvm::parseIR(inputFileBuffer->getMemBufferRef(), error, context); + if (!mod) { + error.print("cross-module-inline", errs()); + errs() << "\n"; + exit(EXIT_FAILURE); + } + return mod; +} +} // anonymous namespace + +// ===================================================================================================================== +// Main code of the testing utility +// +// @param argc : Count of command-line arguments +// @param argv : Command-line arguments +int main(int argc, char **argv) { + const char *progName = sys::path::filename(argv[0]).data(); + + // Parse command line. + static const char *commandDesc = "cross-module-inline: inline from one module into another\n"; + cl::ParseCommandLineOptions(argc, argv, commandDesc); + + // Read input files. + llvm::SMDiagnostic error; + llvm::LLVMContext context; + + auto mainMod = parseIr(context, MainModule); + auto linkMod = parseIr(context, LinkModule); + + CompilerUtils::CrossModuleInliner inliner; + for (auto &linkName : LinkFunction) { + // Search for calls and inline them + auto *linkF = mainMod->getFunction(linkName); + if (!linkF) + report_fatal_error(Twine("Function '") + linkName + "' not found in main module"); + + auto *targetF = linkMod->getFunction(linkName); + if (!targetF) + report_fatal_error(Twine("Function '") + linkName + "' not found in link module"); + + for (auto &Use : make_early_inc_range(linkF->uses())) { + if (auto *CInst = dyn_cast(Use.getUser())) { + if (CInst->isCallee(&Use)) { + // Change call target to other module + Use = targetF; + + inliner.inlineCall(*CInst); + } + } + } + } + + // Output + if (OutFileName.getNumOccurrences() && (OutFileName != "") && (OutFileName != "-")) { + // Write to file + std::error_code errorCode; + llvm::raw_fd_ostream file(OutFileName, errorCode, llvm::sys::fs::OF_Text); + mainMod->print(file, nullptr); + file.close(); + if (errorCode) { + auto error = SMDiagnostic(OutFileName, SourceMgr::DK_Error, "Could not open output file: " + errorCode.message()); + error.print(progName, errs()); + errs() << "\n"; + return EXIT_FAILURE; + } + } else { + // Print to stdout + mainMod->print(llvm::outs(), nullptr); + } + + return EXIT_SUCCESS; +} diff --git a/docs/ComputeShaderDerivateGroups.md b/docs/ComputeShaderDerivateGroups.md new file mode 100644 index 0000000000..782b83be5f --- /dev/null +++ b/docs/ComputeShaderDerivateGroups.md @@ -0,0 +1,128 @@ +# VK_NV_Compute_Shader_derivatives + +# Introduction: +This extension adds Vulkan support for the `SPV_NV_compute_shader_derivatives` SPIR-V extension. + +The SPIR-V extension provides two new execution modes, both of which allow compute shaders to use built-ins that evaluate compute derivatives explicitly or implicitly. + +Derivatives will be computed via difference over a 2x2 group of shader invocations. + +The code:DerivativeGroupQuadsNV execution mode assembles shader invocations into 2x2 groups, compute shader invocations are grouped into 2x2x1 arrays whose four local invocation ID values follow the pattern: + + +-----------------+------------------+ + | (2x+0, 2y+0, z) | (2x+1, 2y+0, z) | + +-----------------+------------------+ + | (2x+0, 2y+1, z) | (2x+1, 2y+1, z) | + +-----------------+------------------+ + + - 0th index has a local invocation ID of the form (2x + 0, 2y + 0, z) + - 1st index has a local invocation ID of the form (2x + 1, 2y + 0, z) + - 2nd index has a local invocation ID of the form (2x + 0, 2y + 1, z) + - 3rd index has a local invocation ID of the form (2x + 1, 2y + 1, z) + +The code:DerivativeGroupLinearNV execution mode assembles shader invocations into 2x2 groups, compute shader invocations are grouped into 2x2x1 arrays whose four local invocation index values follow the pattern: + + +------+------+ + | 4n+0 | 4n+1 | + +------+------+ + | 4n+2 | 4n+3 | + +------+------+ + + - 0th index has a local invocation index of the form 4n + 0 + - 1st index has a local invocation index of the form 4n + 1 + - 2nd index has a local invocation index of the form 4n + 2 + - 3rd index has a local invocation index of the form 4n + 3 + +If neither layout qualifier is specified, derivatives in compute shaders return zero. +Unlike fragment shaders, compute shaders never have any "helper" invocations that are only used for derivatives. As a result, the local work group width and height must be a multiple of two when using the "quads" layout, and the total number of invocations in a local work group must be a multiple of four when using the "linear" layout. + +# BACKGROUND DETAILS +## Derivative Instructions +OpDPdx/OpDPdy/OpFwidth/OpDPdxFine/OpDPdyFine/OpDPdxCoarse/OpDPdyCoarse/OpFwidthCoarse. + +## Derivative Calculation under LLPC +MiscBuilder::CreateDerivative is used as the core function for the implementation. It will use Dpp_Quad_Perm to capture the pixel data across lane to calculate derivative. +The use of ComputeDerivateGroup*NV must affect how LocalInvocationId / LocalInvocationIndex is mapped to SubgroupLocalInvocationId, and so it potentially interacts with reconfigWorkgroupLayout and forceCsThreadIdSwizzling. +Specifically, the WorkgroupLayout needs to be overridden in some cases to be consistent with the ComputeDerivateGroup*NV capability. + +# Implementation +## XGL Change as advertising the extension +### New Macros and variables: +NV_COMPUTESHADER_DERIVATIVES ===> to be added in class DeviceExtensions + +### Structures already existed: +```c++ +typedef struct VkPhysicalDeviceComputeShaderDerivativesFeaturesNV { + VkStructureType sType; + void* pNext; + VkBool32 computeDerivativeGroupQuads; + VkBool32 computeDerivativeGroupLinear; +} VkPhysicalDeviceComputeShaderDerivativesFeaturesNV; +``` + +### Function Support +VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COMPUTE_SHADER_DERIVATIVES_FEATURES_NV should be considered under PhysicalDevice::GetFeatures2() + +## Interface Change +No interface change here + +## LLPC Change +### SPIRV Change: +Updated functions: +SPIRVMap::init() +===> Add two new Capabilities: CapabilityComputeDerivativeGroupLinearNV and CapabilityComputeDerivativeGroupQuadsNV + +SPIRVMap::init() +===> Add support for two new Capabilities +When translate SPIRV to LLVM-IR for derivative OpCode: OpDPdx/OpDPdy/OpDPdxFine/OpDPdyFine/OpDPdxCoarse/OpDPdyCoarse, it also needs to add the support for new Capabilities. + +### LGC Changes +The current support for workgroup layout configuration is messy, with decisions performed in multiple places This should be cleaned up first. The workgroup layout will be reconfigured in the follow situations and the supported layout are listed: + +- PipelineOptions::ReconfigWorkgroupLayout (Calculate workgroup layout according to current work group size) + - Linear (4x1) + - Quads (2x2) + - SexagintiQuads(8x8) +- PipelineOptions:: forceCsThreadIdSwizzling + - 8x4 for wave32 + - 8x8 for wave64 +- ComputeShaderMode::derivatives + - None + - Linear(4x1) + - Quads(2x2) + +The guidance of the swizzle combination: + +1. ComputeShaderMode::derivatives enable, it must override the others when necessary + - None ==> Then consider forceCsThreadIdSwizzling + - Linear ==> Linear must be chosen overall not care other two swizzle functions. + - Quads(2x2) + - forceCsThreadIdSwizzling enable: Macro8xN and Micro2x2 + - reconfigWorkGroupLayout(workGroupSizeX is multiple of 8): Macro8xN and Micro2x2 + - reconfigWorkGroupLayout(both workGroupSizeX and WorkgGroupSizeY are multiple of 2): Micro2x2 +2. ComputeShaderMode::derivative disable OR derivatives is None: + - forceCsThreadId enable: + - WorkGroupSizeX is multiple of 8: Swizzle thread id into layout: Macro8xN + - WorkGroupSizeX is not multiple of 8: Process according to reconfigWorkGroupLayout + - forceCsThreadId disable: + - ReconfigWorkGroupLayout enable: + - WorkGroupSizeX is multiple of 8: Swizzle thread id into layout: Macro8xN + - Both workGroupSizeX and workGroupSizeY are multiple of 2: Micro2x2 + - ReconfigWorkGroupLayout disable: + Nothing to do. cd + +During implementation, some interfaces and structures should be changed as follows: + +1. Replace `ResourceUsage::builtInUsage.cs.workgroupLayout` with a `ResourceUsage::builtInUsage.cs.foldWorkgroupXY` boolean. The `Gfx[69]ConfigBuilder` will use this boolean instead of `workgroupLayout`. +2. Add `ComputeShaderMode::derivatives` field with three enum values: `None`, `Linear`, `Quads`. This field is populated by the SPIRV reader. +3. Change the `InOutBuilder` to always insert a call to `lgc.reconfigure.local.invocation.id` for `BuiltInLocalInvocationId`. Remove `lgc.swizzle.local.invocation.id`. +4. `PatchInOutImportExport::processShader` will handle `lgc.reconfigure.local.invocation.id` similar to today. However: + * Take the `ComputeShaderMode::derivatives` field into account when determining the workgroup layout to use + * Integrate `swizzleLocalInvocationIdIn8x4`; basically, the decision of micro- vs. macro-tiling becomes explicitly orthogonal. Instead of `WorkgroupLayout::{Linear, Quads, SexagintiQuads}` and then 8x4 swizzling, there is only `Linear` vs. `Quads` for the micro-tiling and then `Linear` vs. `Block8` for the macro-tiling. + * The `forceCsThreadIdSwizzling` pipeline option can stay the same for compatibility, it is simply used as an input to the determination of which layout to use. + +Note that all decision-making is centralized in one place, which is `PatchInOutImportExport`: the inputs of pipeline options as well as derivatives mode are used once to determine the workgroup layout. + +# Issues +## Helper invocations support +No support for help invocation on compute shaders. diff --git a/include/vkgcDefs.h b/include/vkgcDefs.h index 8365ffa169..30eb24ae31 100644 --- a/include/vkgcDefs.h +++ b/include/vkgcDefs.h @@ -30,6 +30,7 @@ */ #pragma once +#include "llpcVersion.h" #include "vkgcBase.h" #include "vulkan.h" #include @@ -45,20 +46,6 @@ #undef Bool #endif -/// LLPC major interface version. -#define LLPC_INTERFACE_MAJOR_VERSION 70 - -/// LLPC minor interface version. -#define LLPC_INTERFACE_MINOR_VERSION 2 - -#ifndef LLPC_CLIENT_INTERFACE_MAJOR_VERSION -#error LLPC client version is not defined -#endif - -#if LLPC_CLIENT_INTERFACE_MAJOR_VERSION < 60 -#error LLPC client version is too old -#endif - /// LLPC_NODISCARD - Warns when function return value is discarded. // // We cannot use the 'nodiscard' attribute until we upgrade to C++17 or newer mode. @@ -74,134 +61,6 @@ #define LLPC_NODISCARD #endif -// -// ------------------------------------------------------------------------------------------------------------------- -// @page VersionHistory -// %Version History -// | %Version | Change Description | -// | -------- | ----------------------------------------------------------------------------------------------------- | -// | 70.2 | Add useSoftwareVertexBufferDescriptors to GraphicsPipelineBuildInfo | -// | 70.1 | Add cpsFlags to RayTracingPipelineBuildInfo | -// | 70.0 | Add enablePrimGeneratedQuery to PipelineOptions | -// | 69.1 | Add useBarycentric to ShaderModuleUsage | -// | 69.0 | Enable continuations transform in LLPC | -// | 68.0 | Remove ICache *cache in all PipelineBuildInfo | -// | 67.0 | Modify the uber fetch shader. Adds locationMask(64bit) at the beginning of uber fetch shader internal | -// | | buffer which flags whether the related attribute data is valid. | -// | 66.1 | Add forwardPropagateNoContract and backwardPropagateNoContract to PipelineShaderOptions | -// | 66.0 | Remove shader cache in LLPC | -// | 65.6 | Add Result::RequireFullPipeline, returned if unlink shader fails. | -// | 65.5 | Rename noContract in PipelineShaderOptions to noContractOpDot | -// | 65.4 | Add disableSampleMask to PipelineOptions | -// | 65.3 | Add originUpperLeft to GraphicsPipelineBuildInfo | -// | 65.2 | Support SPIRV extended vertex attribute formats during vertex fetch module. | -// | 65.0 | Remove updateDescInElf | -// | 64.2 | Add dynamicSampleInfo to GraphicsPipelineBuildInfo::rsState | -// | 64.1 | Add disableTruncCoordForGather to PipelineOptions. | -// | 64.0 | Add enableColorExportShader to GraphicsPipelineBuildInfo. | -// | 63.3 | Add TessellationLevel to iaState | -// | 63.2 | Add vertex64BitsAttribSingleLoc to PipelineOptions | -// | 63.1 | Add forceDisableStreamOut and forceEnablePrimStats to ApiXfbOutData | -// | 63.0 | Add Atomic Counter, its default descriptor and map its concreteType to Buffer. | -// | 62.1 | Add ApiXfbOutData GraphicsPipelineBuildInfo | -// | 62.0 | Default to the compiler getting the GPURT library directly, and move shader library info into RtState | -// | 61.16| Add replaceSetWithResourceType to PipelineOptions | -// | 61.15| Add disableReadFirstLaneWorkaround to PipelineShaderOptions | -// | 61.14| Add rasterStream to rsState -// | 61.13| Add dualSourceBlendDynamic to cbState | -// | 61.12| Add mode to RayTracingPipelineBuildInfo | -// | 61.11| Add UniformConstantMap and related structures | -// | 61.10| Add useShadingRate and useSampleInfo to ShaderModuleUsage | -// | 61.8 | Add enableImplicitInvariantExports to PipelineOptions | -// | 61.7 | Add disableFMA to PipelineShaderOptions | -// | 61.6 | Add workaroundInitializeOutputsToZero to PipelineShaderOptions | -// | 61.5 | Add RtIpVersion (including its checkers) to represent RT IP | -// | 61.4 | Add workaroundStorageImageFormats to PipelineShaderOptions | -// | 61.2 | Add pClientMetadata and clientMetadataSize to all PipelineBuildInfos | -// | 61.1 | Add IPipelineDumper::GetGraphicsShaderBinaryHash | -// | 61.0 | Add DescriptorMutable type and ResourceMappingNode::strideInDwords to support mutable descriptors | -// | 60.0 | Simplify the enum NggCompactMode to a boolean flag | -// | 59.0 | Remove the option enableVertexReuse from NggState | -// | 57.2 | Move all internal resource binding id to enum InternalBinding. | -// | 57.1 | Add forceNonUniformResourceIndexStageMask to PipelineOptions | -// | 57.0 | Merge aggressiveInvariantLoads and disableInvariantLoads to an enumerated option | -// | 56.2 | Add aggressiveInvariantLoads and disableInvariantLoads to PipelineShaderOptions | -// | 56.1 | Add struct UberFetchShaderAttribInfo | -// | 56.0 | Move maxRayLength to RtState | -// | 55.2 | Add pipeline layout API hash to all PipelineBuildInfos | -// | 55.1 | Add nsaThreshold to PipelineShaderOptions | -// | 55.0 | Remove isInternalRtShader from module options | -// | 54.9 | Add internalRtShaders to PipelineOptions to allow for dumping this data | -// | 54.6 | Add reverseThreadGroup to PipelineOptions | -// | 54.5 | Add forceLateZ to PipelineShaderOptions | -// | 54.4 | Add isReplay to RayTracingPipelineBuildInfo for ray tracing capture replay feature | -// | 54.3 | Add usePointSize to ShaderModuleUsage | -// | 54.2 | Add subgroupSize to PipelineShaderOptions | -// | 54.1 | Add overrideForceThreadIdSwizzling overrideShaderThreadGroupSizeX, overrideShaderThreadGroupSizeY | -// | | and overrideShaderThreadGroupSizeZ to PipelineShaderOptions | -// | 54.0 | Add overrideThreadGroupSizeX, overrideThreadGroupSizeY and overrideThreadGroupSizeZ to PipelineOptions| -// | 53.7 | Add threadGroupSwizzleMode to PipelineOptions | -// | 53.6 | Add scalarizeWaterfallLoads to PipelineShaderOptions | -// | 53.5 | Add forceCsThreadIdSwizzling for thread id swizzle in 8*4 | -// | 53.4 | Add ldsSpillLimitDwords shader option | -// | 53.3 | Add disableFastMathFlags shader option, plus support for this and fastMathFlags in pipeline files | -// | 53.2 | Add resourceLayoutScheme to PipelineOptions | -// | 53.1 | Add PartPipelineStage enum for part-pipeline mode | -// | 53.0 | Add optimizationLevel to PipelineOptions | -// | 52.3 | Add fastMathFlags to PipelineShaderOptions | -// | 52.2 | Add provokingVertexMode to rsState | -// | 52.1 | Add pageMigrationEnabled to PipelineOptions | -// | 52.0 | Add the member word4 and word5 to SamplerYCbCrConversionMetaData | -// | 51.2 | Added new pipeline shader info to support mesh shader | -// | 51.0 | Added new shader stage enumerates to support mesh shader | -// | 50.2 | Add the member dsState to GraphicsPipelineBuildInfo | -// | 50.1 | Disclose ResourceMappingNodeType::InlineBuffer | -// | 50.0 | Removed the member 'enableOpt' of ShaderModuleOptions | -// | 49.1 | Added enableEarlyCompile to GraphicsPipelineBuildInfo | -// | 49.0 | Added DescriptorConstBuffer, DescriptorConstBufferCompact, DescriptorImage, DescriptorConstTexelBuffer| -// | | to ResourceMappingNodeType | -// | 48.1 | Added enableUberFetchShader to GraphicsPipelineBuildInfo | -// | 48.0 | Removed the member 'polygonMode' of rsState | -// | 47.0 | Always get culling controls from primitive shader table | -// | 46.3 | Added enableInterpModePatch to PipelineOptions | -// | 46.2 | Added optimizeTessFactor to PipelineOptions | -// | 46.1 | Added dynamicVertexStride to GraphicsPipelineBuildInfo | -// | 46.0 | Removed the member 'depthBiasEnable' of rsState | -// | 45.5 | Added new enum type ThreadGroupSwizzleMode for thread group swizzling for compute shaders | -// | 45.4 | Added disableLicmThreshold, unrollHintThreshold, and dontUnrollHintThreshold to PipelineShaderOptions | -// | 45.3 | Add pipelinedump function to enable BeginPipelineDump and GetPipelineName | -// | 45.2 | Add GFX IP plus checker to GfxIpVersion | -// | 45.1 | Add pipelineCacheAccess, stageCacheAccess(es) to GraphicsPipelineBuildOut/ComputePipelineBuildOut | -// | 45.0 | Remove the member 'enableFastLaunch' of NGG state | -// | 44.0 | Rename the member 'forceNonPassthrough' of NGG state to 'forceCullingMode' | -// | 43.1 | Add disableImageResourceCheck in PipelineOptions | -// | 43.0 | Removed the enumerate WaveBreakSize::DrawTime | -// | 42.0 | Removed tileOptimal flag from SamplerYcbcrConversion metadata struct | -// | 41.0 | Moved resource mapping from ShaderPipeline-level to Pipeline-level | -// | 40.4 | Added fp32DenormalMode in PipelineShaderOptions to allow overriding SPIR-V denormal settings | -// | 40.3 | Added ICache interface | -// | 40.2 | Added extendedRobustness in PipelineOptions to support VK_EXT_robustness2 | -// | 40.1 | Added disableLoopUnroll to PipelineShaderOptions | -// | 40.0 | Added DescriptorReserved12, which moves DescriptorYCbCrSampler down to 13 | -// | 39.0 | Non-LLPC-specific XGL code should #include vkgcDefs.h instead of llpc.h | -// | 38.3 | Added shadowDescriptorTableUsage and shadowDescriptorTablePtrHigh to PipelineOptions | -// | 38.2 | Added scalarThreshold to PipelineShaderOptions | -// | 38.1 | Added unrollThreshold to PipelineShaderOptions | -// | 38.0 | Removed CreateShaderCache in ICompiler and pShaderCache in pipeline build info | -// | 37.0 | Removed the -enable-dynamic-loop-unroll option | -// | 36.0 | Add 128 bit hash as clientHash in PipelineShaderOptions | -// | 35.0 | Added disableLicm to PipelineShaderOptions | -// | 33.0 | Add enableLoadScalarizer option into PipelineShaderOptions. | -// | 32.0 | Add ShaderModuleOptions in ShaderModuleBuildInfo | -// | 31.0 | Add PipelineShaderOptions::allowVaryWaveSize | -// | 30.0 | Removed PipelineOptions::autoLayoutDesc | -// | 28.0 | Added reconfigWorkgroupLayout to PipelineOptions and useSiScheduler to PipelineShaderOptions | -// | 27.0 | Remove the includeIrBinary option from PipelineOptions as only IR disassembly is now dumped | -// | 25.0 | Add includeIrBinary option into PipelineOptions for including IR binaries into ELF files. | -// | 24.0 | Add forceLoopUnrollCount option into PipelineShaderOptions. | -// | 23.0 | Add flag robustBufferAccess in PipelineOptions to check out of bounds of private array. | -// | 22.0 | Internal revision. | -// | 21.0 | Add stage in Pipeline shader info and struct PipelineBuildInfo to simplify pipeline dump interface. | // // IMPORTANT NOTE: All structures defined in this file that are passed as input into LLPC must be zero-initialized // with code such as the following before filling in the structure's fields: @@ -610,6 +469,8 @@ struct PipelineOptions { ///< for sampled images and samplers bool vertex64BitsAttribSingleLoc; ///< For OGL only, dvec3/dvec4 vertex attrib only consumes 1 location. bool enableFragColor; ///< For OGL only, need to do frag color broadcast if it is enabled. + bool disableBaseVertex; ///< For OGL only, force the BaseVertex builtin to 0 instead of + /// loading it from userdata unsigned reserved20; bool enablePrimGeneratedQuery; ///< If set, primitive generated query is enabled }; diff --git a/lgc/CMakeLists.txt b/lgc/CMakeLists.txt index d6ae3674a3..8cde640507 100644 --- a/lgc/CMakeLists.txt +++ b/lgc/CMakeLists.txt @@ -54,19 +54,9 @@ add_llvm_library(LLVMlgc LINK_COMPONENTS ) llvm_map_components_to_libnames(extra_llvm_libs CompilerUtils Continuations) -target_link_libraries(LLVMlgc PUBLIC ${extra_llvm_libs}) +target_link_libraries(LLVMlgc PUBLIC llvm_dialects ${extra_llvm_libs} llpc_version) ### Cached Project Options ############################################################################################# -option(LLPC_BUILD_NAVI12 "LLPC support for NAVI12?" ON) -#if VKI_BUILD_NAVI32 -option(LLPC_BUILD_NAVI32 "LLPC support for NAVI32?" ON) -#endif -option(LLPC_BUILD_REMBRANDT "LLPC support for REMBRANDT?" ON) -option(LLPC_BUILD_RAPHAEL "LLPC support for RAPHAEL?" ON) -option(LLPC_BUILD_MENDOCINO "LLPC support for MENDOCINO?" ON) -#if VKI_BUILD_PHOENIX1 -option(LLPC_BUILD_PHOENIX1 "LLPC support for PHOENIX1?" ON) -#endif option(LLPC_ENABLE_WERROR "Build LLPC with more errors" OFF) ### Compiler Options ################################################################################################### @@ -97,64 +87,6 @@ add_public_tablegen_target(LgcDialectTableGen) add_dependencies(LLVMlgc LgcDialectTableGen) ### Defines/Includes/Sources ########################################################################################### -target_compile_definitions(LLVMlgc PRIVATE ${TARGET_ARCHITECTURE_ENDIANESS}ENDIAN_CPU) -if (LLPC_CLIENT_INTERFACE_MAJOR_VERSION) - target_compile_definitions(LLVMlgc PRIVATE LLPC_CLIENT_INTERFACE_MAJOR_VERSION=${LLPC_CLIENT_INTERFACE_MAJOR_VERSION}) -endif() -target_compile_definitions(LLVMlgc PRIVATE - LITTLEENDIAN_CPU - CHIP_HDR_NAVI14 - CHIP_HDR_NAVI21 - CHIP_HDR_NAVI22 - CHIP_HDR_NAVI23 - CHIP_HDR_NAVI24 - CHIP_HDR_NAVI31 - CHIP_HDR_NAVI33 - ) - -target_compile_definitions(LLVMlgc PRIVATE CHIP_HDR_RENOIR) - - if(LLPC_BUILD_NAVI12) - target_compile_definitions(LLVMlgc PRIVATE - LLPC_BUILD_NAVI12 - CHIP_HDR_NAVI12 - ) - endif() -#if VKI_BUILD_NAVI32 - if(LLPC_BUILD_NAVI32) - target_compile_definitions(LLVMlgc PRIVATE - LLPC_BUILD_NAVI32 - CHIP_HDR_NAVI32 - ) - endif() -#endif - if(LLPC_BUILD_REMBRANDT) - target_compile_definitions(LLVMlgc PRIVATE - LLPC_BUILD_REMBRANDT - CHIP_HDR_REMBRANDT - ) - endif() - if(LLPC_BUILD_RAPHAEL) - target_compile_definitions(LLVMlgc PRIVATE - LLPC_BUILD_RAPHAEL - CHIP_HDR_RAPHAEL - ) - endif() - if(LLPC_BUILD_MENDOCINO) - target_compile_definitions(LLVMlgc PRIVATE - LLPC_BUILD_MENDOCINO - CHIP_HDR_MENDOCINO - ) - endif() -#if VKI_BUILD_PHOENIX1 - if(LLPC_BUILD_PHOENIX1) - target_compile_definitions(LLVMlgc PRIVATE - LLPC_BUILD_PHOENIX1 - CHIP_HDR_PHOENIX1 - ) - endif() -#endif - if(WIN32) target_compile_definitions(LLVMlgc PRIVATE NOMINMAX # windows.h defines min/max which conflicts with the use of std::min / max diff --git a/lgc/builder/BuilderRecorder.cpp b/lgc/builder/BuilderRecorder.cpp index 2908eef1d3..179d9690fe 100644 --- a/lgc/builder/BuilderRecorder.cpp +++ b/lgc/builder/BuilderRecorder.cpp @@ -308,6 +308,8 @@ StringRef BuilderRecorder::getCallName(BuilderOpcode opcode) { return "subgroup.clustered.inclusive"; case BuilderOpcode::SubgroupClusteredExclusive: return "subgroup.clustered.exclusive"; + case BuilderOpcode::SubgroupClusteredMultiExclusive: + return "subgroup.clustered.multi.exclusive"; case BuilderOpcode::SubgroupQuadBroadcast: return "subgroup.quad.broadcast"; case BuilderOpcode::SubgroupQuadSwapHorizontal: @@ -324,6 +326,8 @@ StringRef BuilderRecorder::getCallName(BuilderOpcode opcode) { return "subgroup.write.invocation"; case BuilderOpcode::SubgroupMbcnt: return "subgroup.mbcnt"; + case BuilderOpcode::SubgroupPartition: + return "subgroup.partition"; case BuilderOpcode::Count: break; } @@ -1872,6 +1876,19 @@ Value *Builder::CreateSubgroupClusteredExclusive(GroupArithOp groupArithOp, Valu {getInt32(groupArithOp), value, clusterSize}, instName); } +// ===================================================================================================================== +// Create a subgroup clustered multi exclusive scan. +// +// @param groupArithOp : The group operation to perform +// @param value : The value to perform on +// @param mask : The mask of each exclusive +// @param instName : Name to give instruction(s) +Value *Builder::CreateSubgroupClusteredMultiExclusive(GroupArithOp groupArithOp, Value *const value, Value *const mask, + const Twine &instName) { + return record(BuilderOpcode::SubgroupClusteredMultiExclusive, value->getType(), {getInt32(groupArithOp), value, mask}, + instName); +} + // ===================================================================================================================== // Create a subgroup quad broadcast. // @@ -1952,6 +1969,15 @@ Value *Builder::CreateSubgroupMbcnt(Value *const mask, const Twine &instName) { return record(BuilderOpcode::SubgroupMbcnt, getInt32Ty(), mask, instName); } +// ===================================================================================================================== +// Create a subgroup partition. +// +// @param value : The value to contribute +// @param instName : Name to give instruction(s) +Value *Builder::CreateSubgroupPartition(Value *const value, const Twine &instName) { + return record(BuilderOpcode::SubgroupPartition, FixedVectorType::get(getInt32Ty(), 4), value, instName); +} + // ===================================================================================================================== // Record one Builder call // @@ -2105,11 +2131,13 @@ Instruction *Builder::record(BuilderOpcode opcode, Type *resultTy, ArrayRef(cast(args[0])->getZExtValue()); return m_builder->CreateSubgroupClusteredExclusive(groupArithOp, args[1], args[2]); } + case BuilderOpcode::SubgroupClusteredMultiExclusive: { + Builder::GroupArithOp groupArithOp = static_cast(cast(args[0])->getZExtValue()); + return m_builder->CreateSubgroupClusteredMultiExclusive(groupArithOp, args[1], args[2]); + } case BuilderOpcode::SubgroupQuadBroadcast: { return m_builder->CreateSubgroupQuadBroadcast(args[0], args[1], cast(args[2])->getZExtValue()); } @@ -831,5 +835,8 @@ Value *BuilderReplayer::processCall(unsigned opcode, CallInst *call) { case BuilderOpcode::SubgroupMbcnt: { return m_builder->CreateSubgroupMbcnt(args[0]); } + case BuilderOpcode::SubgroupPartition: { + return m_builder->CreateSubgroupPartition(args[0]); + } } } diff --git a/lgc/builder/InOutBuilder.cpp b/lgc/builder/InOutBuilder.cpp index 3c20bbe2f4..1b9995c30f 100644 --- a/lgc/builder/InOutBuilder.cpp +++ b/lgc/builder/InOutBuilder.cpp @@ -484,10 +484,6 @@ void BuilderImpl::markGenericInputOutputUsage(bool isOutput, unsigned location, // Mark usage for interpolation info. markInterpolationInfo(inOutInfo); } - - if (isOutput && m_shaderStage == ShaderStageFragment && inOutInfo.isDualSourceBlendDynamic()) { - m_pipelineState->getColorExportState().dynamicDualSourceBlendEnable = true; - } } // ===================================================================================================================== @@ -1000,6 +996,7 @@ void BuilderImpl::getProvokingVertexInfo(llvm::Value **isOne, llvm::Value **isTw auto provokingVtxInfo = ShaderInputs::getInput(ShaderInput::ProvokingVtxInfo, BuilderBase::get(*this), *getLgcContext()); +#if LLVM_MAIN_REVISION && LLVM_MAIN_REVISION < 479645 auto laneID = CreateGetLaneNumber(); auto quadId = CreateSDiv(laneID, getInt32(4)); auto provokingVertex = CreateIntrinsic(Intrinsic::amdgcn_ubfe, getInt32Ty(), @@ -1008,16 +1005,18 @@ void BuilderImpl::getProvokingVertexInfo(llvm::Value **isOne, llvm::Value **isTw *isOne = CreateICmpEQ(provokingVertex, getInt32(1)); *isTwo = CreateICmpEQ(provokingVertex, getInt32(2)); - // TODO: Here should have a better implementation by staying in "lane mask space" and using the fact that the - // provoking vertex info is always either 0, 1, or 2. Unfortunately, S_BITREPLICATE was not exposed. - // - // isTwoMask = provoking_vtx_info & 0xaaaaaaaa - // isOneMask = (provoking_vtx_info & 0x55555555) & ~(isTwoMask >> 1) - // isTwoMask = llvm.amdgcn.wqm.vote(S_BITREPLICATE_B64_B32(isTwoMask)) - // isOneMask = llvm.amdgcn.wqm.vote(S_BITREPLICATE_B64_B32(isOneMask)) - // isTwo = llvm.amdgcn.inverse.ballot(isTwoMask) - // isOne = llvm.amdgcn.inverse.ballot(isOneMask) - // +#else + // Extract 2-bit vertex index from provokingVtxInfo + auto isTwoMask = CreateAnd(provokingVtxInfo, getInt32(0xAAAAAAAA)); + auto isOneMask = + CreateAnd(CreateAnd(provokingVtxInfo, getInt32(0x55555555)), CreateLShr(CreateNot(isTwoMask), getInt32(1))); + isTwoMask = CreateIntrinsic(Intrinsic::amdgcn_s_bitreplicate, {}, isTwoMask); + isOneMask = CreateIntrinsic(Intrinsic::amdgcn_s_bitreplicate, {}, isOneMask); + isTwoMask = CreateIntrinsic(getInt64Ty(), Intrinsic::amdgcn_s_wqm, isTwoMask); + isOneMask = CreateIntrinsic(getInt64Ty(), Intrinsic::amdgcn_s_wqm, isOneMask); + *isTwo = CreateIntrinsic(getInt1Ty(), Intrinsic::amdgcn_inverse_ballot, isTwoMask); + *isOne = CreateIntrinsic(getInt1Ty(), Intrinsic::amdgcn_inverse_ballot, isOneMask); +#endif } // ===================================================================================================================== diff --git a/lgc/builder/MatrixBuilder.cpp b/lgc/builder/MatrixBuilder.cpp index fd16770333..ac70c653a7 100644 --- a/lgc/builder/MatrixBuilder.cpp +++ b/lgc/builder/MatrixBuilder.cpp @@ -87,7 +87,8 @@ Value *BuilderImpl::CreateTransposeMatrix(Value *const matrix, const Twine &inst // Create matrix from matrix Times scalar // // @param matrix : The column major matrix, n x -// @param scalar : The float scalar +// @param scalar : The float scalar. If the matrix is a packed accumulator +// matrix, the scalar has to be a <2 x half> vector. // @param instName : Name to give instruction(s) Value *BuilderImpl::CreateMatrixTimesScalar(Value *const matrix, Value *const scalar, const Twine &instName) { Type *const matrixTy = matrix->getType(); @@ -351,6 +352,7 @@ Value *BuilderImpl::CreateMatrixInverse(Value *const matrix, const Twine &instNa Type *BuilderCommon::transCooperativeMatrixElementType(CooperativeMatrixElementType elemType) { switch (elemType) { case BuilderCommon::CooperativeMatrixElementType::Float16: + case BuilderCommon::CooperativeMatrixElementType::Float16Packed: return getHalfTy(); case BuilderCommon::CooperativeMatrixElementType::Float32: return getFloatTy(); @@ -585,14 +587,17 @@ Value *BuilderCommon::CreateCooperativeMatrixBinaryOp(CooperativeMatrixArithOp c // Create cooperative matrix MatrixTimesScalar operation // // @param matrix : The first operand and it should be a cooperative matrix. -// @param scalar : The second operand and it should be a scalar. +// @param scalar : The second operand and it should be a scalar. If the matrix is a packed accumulator matrix, the +// scalar has to be a <2 x half> vector. // @param elemType : The component type of the matrix. // @param layout : Identify whether it's A/B or C/D // @param instName : Name to give instruction(s). Value *BuilderCommon::CreateCoopMatrixTimesScalar(Value *matrix, Value *scalar, CooperativeMatrixElementType elemType, CooperativeMatrixLayout layout, const Twine &instName) { assert(matrix->getType() == getCooperativeMatrixTy(elemType, layout)); - assert(scalar->getType() == transCooperativeMatrixElementType(elemType)); + assert(scalar->getType() == (elemType == CooperativeMatrixElementType::Float16Packed + ? FixedVectorType::get(getHalfTy(), 2) + : transCooperativeMatrixElementType(elemType))); std::string callName(lgcName::CooperativeMatrixTimesScalar); Value *args[] = {matrix, scalar, getInt32(static_cast(elemType)), getInt32(static_cast(layout))}; @@ -631,11 +636,14 @@ CallInst *BuilderCommon::CreateCooperativeMatrixTranspose(llvm::Value *matrix, C // @param matrixC : Accumulator cooperative matrix. // @param isSignedA : Identify the signess for matrix A's element type // @param isSignedB : Identify the signess for matrix B's element type -// @param isSat : SaturatingAccumulation for calculation +// @param isSatOrOpsel : SaturatingAccumulation for calculation. In the case of 16-bit floating point +// matrices, this bit acts as an opsel bit. If it is set to false, we store the result in the lower half of +// the registers. If it is true, we store it in the upper half. +// @param isTied : If true, the output matrix has to be the same as the input accumulator (i.e., D has to be C) // @param accumElemType : The component type of the accumulator matrix. // @param factorElemType : The component type of the factor matrix. Value *BuilderCommon::CreateCooperativeMatrixMulAdd(llvm::Value *matrixA, llvm::Value *matrixB, llvm::Value *matrixC, - bool isSignedA, bool isSignedB, bool isSat, + bool isSignedA, bool isSignedB, bool isSatOrOpsel, bool isTied, CooperativeMatrixElementType accumElemType, CooperativeMatrixElementType factorElemType, const llvm::Twine &instName) { @@ -645,7 +653,8 @@ Value *BuilderCommon::CreateCooperativeMatrixMulAdd(llvm::Value *matrixA, llvm:: matrixC, getInt1(isSignedA), getInt1(isSignedB), - getInt1(isSat), + getInt1(isSatOrOpsel), + getInt1(isTied), getInt32(static_cast(accumElemType)), getInt32(static_cast(factorElemType))}; addTypeMangling(matrixC->getType(), args, callName); @@ -654,3 +663,39 @@ Value *BuilderCommon::CreateCooperativeMatrixMulAdd(llvm::Value *matrixA, llvm:: result->setName(instName); return result; } + +// ===================================================================================================================== +// Create cooperative matrix pack operation +// +// @param matrixCLo : Lower Accumulator cooperative matrix. +// @param matrixCHi : Upper Accumulator cooperative matrix. +Value *BuilderCommon::CreateCooperativeMatrixPack(llvm::Value *matrixCLo, llvm::Value *matrixCHi, + const llvm::Twine &instName) { + + std::string callName(lgcName::CooperativeMatrixPack); + Value *args[] = {matrixCLo, matrixCHi}; + + Type *retTy = matrixCLo->getType(); + addTypeMangling(retTy, args, callName); + + Value *result = CreateNamedCall(callName, retTy, args, {Attribute::ReadNone, Attribute::WillReturn}); + result->setName(instName); + return result; +} + +// ===================================================================================================================== +// Create cooperative matrix unpack operation +// +// @param packedMatrix : Packed Accumulator cooperative matrix. +// @param high: Whether to get the matrix stored in the upper half of the registers. +Value *BuilderCommon::CreateCooperativeMatrixUnpack(llvm::Value *packedMatrix, bool high, const llvm::Twine &instName) { + + std::string callName(lgcName::CooperativeMatrixUnpack); + Value *args[] = {packedMatrix, getInt1(high)}; + Type *retTy = packedMatrix->getType(); + addTypeMangling(retTy, args, callName); + + Value *result = CreateNamedCall(callName, retTy, args, {Attribute::ReadNone, Attribute::WillReturn}); + result->setName(instName); + return result; +} diff --git a/lgc/builder/SubgroupBuilder.cpp b/lgc/builder/SubgroupBuilder.cpp index a7a9da5303..b7279a3142 100644 --- a/lgc/builder/SubgroupBuilder.cpp +++ b/lgc/builder/SubgroupBuilder.cpp @@ -33,6 +33,7 @@ #include "lgc/util/Internal.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/IntrinsicsAMDGPU.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" #define DEBUG_TYPE "lgc-builder-impl-subgroup" @@ -89,7 +90,8 @@ Value *BuilderImpl::CreateSubgroupAll(Value *const value, const Twine &instName) result = CreateSelect(CreateUnaryIntrinsic(Intrinsic::is_constant, value), value, result); // Helper invocations of whole quad mode should be included in the subgroup vote execution - if (m_shaderStage == ShaderStageFragment) { + const auto &fragmentMode = m_pipelineState->getShaderModes()->getFragmentShaderMode(); + if (m_shaderStage == ShaderStageFragment && !fragmentMode.waveOpsExcludeHelperLanes) { result = CreateZExt(result, getInt32Ty()); result = CreateIntrinsic(Intrinsic::amdgcn_softwqm, {getInt32Ty()}, {result}); result = CreateTrunc(result, getInt1Ty()); @@ -107,7 +109,8 @@ Value *BuilderImpl::CreateSubgroupAny(Value *const value, const Twine &instName) result = CreateSelect(CreateUnaryIntrinsic(Intrinsic::is_constant, value), value, result); // Helper invocations of whole quad mode should be included in the subgroup vote execution - if (m_shaderStage == ShaderStageFragment) { + const auto &fragmentMode = m_pipelineState->getShaderModes()->getFragmentShaderMode(); + if (m_shaderStage == ShaderStageFragment && !fragmentMode.waveOpsExcludeHelperLanes) { result = CreateZExt(result, getInt32Ty()); result = CreateIntrinsic(Intrinsic::amdgcn_softwqm, {getInt32Ty()}, {result}); result = CreateTrunc(result, getInt1Ty()); @@ -203,6 +206,22 @@ Value *BuilderImpl::CreateSubgroupBroadcastWaterfall(Value *const value, Value * // @param value : The value to read from the first active lane into all other active lanes. // @param instName : Name to give final instruction. Value *BuilderImpl::CreateSubgroupBroadcastFirst(Value *const value, const Twine &instName) { + const auto &fragmentMode = m_pipelineState->getShaderModes()->getFragmentShaderMode(); + // For waveOpsExcludeHelperLanes mode, we need filter out the helperlane and use readlane instead. + if (m_shaderStage == ShaderStageFragment && fragmentMode.waveOpsExcludeHelperLanes) { + Value *ballot = createGroupBallot(getTrue()); + Value *firstlane = CreateIntrinsic(Intrinsic::cttz, getInt64Ty(), {ballot, getTrue()}); + firstlane = CreateTrunc(firstlane, getInt32Ty()); + + auto mapFunc = [](BuilderBase &builder, ArrayRef mappedArgs, + ArrayRef passthroughArgs) -> Value * { + return builder.CreateIntrinsic(builder.getInt32Ty(), Intrinsic::amdgcn_readlane, + {mappedArgs[0], passthroughArgs[0]}); + }; + + return CreateMapToSimpleType(mapFunc, value, firstlane); + } + auto mapFunc = [](BuilderBase &builder, ArrayRef mappedArgs, ArrayRef passthroughArgs) -> Value * { return builder.CreateIntrinsic(builder.getInt32Ty(), Intrinsic::amdgcn_readfirstlane, mappedArgs[0]); }; @@ -535,6 +554,13 @@ Value *BuilderImpl::CreateSubgroupClusteredReduction(GroupArithOp groupArithOp, Value *const identity = createGroupArithmeticIdentity(groupArithOp, value->getType()); Value *result = BuilderBase::get(*this).CreateSetInactive(value, identity); + // For waveOpsExcludeHelperLanes mode, we need mask away the helperlane. + const auto &fragmentMode = m_pipelineState->getShaderModes()->getFragmentShaderMode(); + if (m_shaderStage == ShaderStageFragment && fragmentMode.waveOpsExcludeHelperLanes) { + auto isLive = CreateIntrinsic(Intrinsic::amdgcn_live_mask, {}, {}, nullptr, {}); + result = CreateSelect(isLive, result, identity); + } + // Perform The group arithmetic operation between adjacent lanes in the subgroup, with all masks and rows enabled // (0xF). result = CreateSelect( @@ -726,7 +752,14 @@ Value *BuilderImpl::CreateSubgroupClusteredExclusive(GroupArithOp groupArithOp, Value *const identity = createGroupArithmeticIdentity(groupArithOp, value->getType()); // Start the WWM section by setting the inactive invocations. - Value *const setInactive = BuilderBase::get(*this).CreateSetInactive(value, identity); + Value *setInactive = BuilderBase::get(*this).CreateSetInactive(value, identity); + + // For waveOpsExcludeHelperLanes mode, we need mask away the helperlane. + const auto &fragmentMode = m_pipelineState->getShaderModes()->getFragmentShaderMode(); + if (m_shaderStage == ShaderStageFragment && fragmentMode.waveOpsExcludeHelperLanes) { + auto isLive = CreateIntrinsic(Intrinsic::amdgcn_live_mask, {}, {}, nullptr, {}); + setInactive = CreateSelect(isLive, setInactive, identity); + } Value *shiftRight = nullptr; @@ -833,6 +866,77 @@ Value *BuilderImpl::CreateSubgroupClusteredExclusive(GroupArithOp groupArithOp, return createWwm(result); } +// ===================================================================================================================== +// Create a subgroup clustered exclusive scan. +// +// @param groupArithOp : The group arithmetic operation. +// @param value : An LLVM value. +// @param mask : Mask of the cluster. Must be <4 x i32> type. +// @param instName : Name to give final instruction. +Value *BuilderImpl::CreateSubgroupClusteredMultiExclusive(GroupArithOp groupArithOp, Value *const value, + Value *const mask, const Twine &instName) { + Value *const identity = createGroupArithmeticIdentity(groupArithOp, value->getType()); + + Value *laneIndex = CreateGetLaneNumber(); + Value *clusterMask = + (getShaderSubgroupSize() <= 32) + ? CreateExtractElement(mask, getInt32(0)) + : CreateBitCast(CreateShuffleVector(mask, PoisonValue::get(mask->getType()), ArrayRef{0, 1}), + getInt64Ty()); + + Value *result = value; + + // For waveOpsExcludeHelperLanes mode, we need mask away the helperlane. + const auto &fragmentMode = m_pipelineState->getShaderModes()->getFragmentShaderMode(); + if (m_shaderStage == ShaderStageFragment && fragmentMode.waveOpsExcludeHelperLanes) { + auto isLive = CreateIntrinsic(Intrinsic::amdgcn_live_mask, {}, {}, nullptr, {}); + result = CreateSelect(isLive, result, identity); + } + + Value *constOne = ConstantInt::get(clusterMask->getType(), 1); + Value *constZero = ConstantInt::get(clusterMask->getType(), 0); + + // Move current lane to next lane according to the mask = (1 << laneIndex) -1. + Value *preLaneMask = CreateSub(CreateShl(constOne, CreateZExtOrTrunc(laneIndex, clusterMask->getType())), constOne); + Value *checkMask = CreateAnd(preLaneMask, clusterMask); + + Value *preLaneValue = CreateSubgroupShuffle(result, createFindMsb(checkMask), instName); + result = CreateSelect(CreateICmpNE(checkMask, constZero), preLaneValue, identity); + + for (unsigned log2ClusterSize = 0; (1 << log2ClusterSize) < getShaderWaveSize(); log2ClusterSize++) { + unsigned clusterSize = 1 << log2ClusterSize; + // Generate the mask of previous cluster before the current lane. Zero for overflow index shift left. + // mask = ((1 << clusterSize) - 1) << ((laneIndex - clusterSize) & ~(clusterSize - 1)). + Value *laneIndexOfCluster = CreateAnd(CreateSub(laneIndex, getInt32(clusterSize)), getInt32(~(clusterSize - 1))); + Value *preClusterMask = CreateShl(ConstantInt::get(clusterMask->getType(), (1ull << clusterSize) - 1), + CreateZExtOrTrunc(laneIndexOfCluster, clusterMask->getType())); + preClusterMask = CreateAnd(preClusterMask, clusterMask); + + Value *isPreviousLaneValid = CreateICmpNE(preClusterMask, constZero); + Value *previousLaneIndex = createFindMsb(preClusterMask); + Value *previousLaneValue = CreateSubgroupShuffle(result, previousLaneIndex, instName); + + // Don't accumulate if there is no valid lane found in previous cluster or current lane is no need for accumulate. +#if LLVM_MAIN_REVISION && LLVM_MAIN_REVISION < 479645 + Value *isAccumulateLane = CreateICmpNE(CreateAnd(laneIndex, getInt32(clusterSize)), getInt32(0)); +#else + // TODO: Check amdgcn_inverse_ballot version. + const unsigned long long halfMasks[] = { + 0x5555555555555555ull, 0x3333333333333333ull, 0x0f0f0f0f0f0f0f0full, + 0x00ff00ff00ff00ffull, 0x0000ffff0000ffffull, 0x00000000ffffffffull, + }; + + Value *isAccumulateLane = CreateIntrinsic(getInt1Ty(), Intrinsic::amdgcn_inverse_ballot, + ConstantInt::get(clusterMask->getType(), ~halfMasks[log2ClusterSize])); +#endif + + previousLaneValue = CreateSelect(CreateAnd(isAccumulateLane, isPreviousLaneValid), previousLaneValue, identity); + result = createGroupArithmeticOperation(groupArithOp, result, previousLaneValue); + } + + return result; +} + // ===================================================================================================================== // Create a quad broadcast call. // @@ -845,18 +949,19 @@ Value *BuilderImpl::CreateSubgroupQuadBroadcast(Value *const value, Value *const Value *result = PoisonValue::get(value->getType()); const unsigned indexBits = index->getType()->getPrimitiveSizeInBits(); + { + Value *compare = CreateICmpEQ(index, getIntN(indexBits, 0)); + result = CreateSelect(compare, createDppMov(value, DppCtrl::DppQuadPerm0000, 0xF, 0xF, true), result); - Value *compare = CreateICmpEQ(index, getIntN(indexBits, 0)); - result = CreateSelect(compare, createDppMov(value, DppCtrl::DppQuadPerm0000, 0xF, 0xF, true), result); - - compare = CreateICmpEQ(index, getIntN(indexBits, 1)); - result = CreateSelect(compare, createDppMov(value, DppCtrl::DppQuadPerm1111, 0xF, 0xF, true), result); + compare = CreateICmpEQ(index, getIntN(indexBits, 1)); + result = CreateSelect(compare, createDppMov(value, DppCtrl::DppQuadPerm1111, 0xF, 0xF, true), result); - compare = CreateICmpEQ(index, getIntN(indexBits, 2)); - result = CreateSelect(compare, createDppMov(value, DppCtrl::DppQuadPerm2222, 0xF, 0xF, true), result); + compare = CreateICmpEQ(index, getIntN(indexBits, 2)); + result = CreateSelect(compare, createDppMov(value, DppCtrl::DppQuadPerm2222, 0xF, 0xF, true), result); - compare = CreateICmpEQ(index, getIntN(indexBits, 3)); - result = CreateSelect(compare, createDppMov(value, DppCtrl::DppQuadPerm3333, 0xF, 0xF, true), result); + compare = CreateICmpEQ(index, getIntN(indexBits, 3)); + result = CreateSelect(compare, createDppMov(value, DppCtrl::DppQuadPerm3333, 0xF, 0xF, true), result); + } if (inWqm) result = createWqm(result); @@ -963,6 +1068,50 @@ Value *BuilderImpl::CreateSubgroupMbcnt(Value *const mask, const Twine &instName return CreateIntrinsic(Intrinsic::amdgcn_mbcnt_hi, {}, {maskHigh, mbcntLo}); } +// ===================================================================================================================== +// Create a subgroup partition. +// +// @param value : The value to partition with. +// @param instName : Name to give instruction(s) +Value *BuilderImpl::CreateSubgroupPartition(llvm::Value *const value, const Twine &instName) { + BasicBlock *currentBlock = GetInsertBlock(); + auto insertPoint = GetInsertPoint(); + + BasicBlock *beforeBlock = + splitBlockBefore(currentBlock, &*insertPoint, nullptr, nullptr, nullptr, currentBlock->getName()); + BasicBlock *loopBlock = splitBlockBefore(currentBlock, &*insertPoint, nullptr, nullptr, nullptr, "loop.body"); + BasicBlock *loopEndBlock = currentBlock; + + // Handle before Block + SetInsertPoint(beforeBlock->getTerminator()); + // bitcase float to uint. + Value *targetValue = value; + if (targetValue->getType() == getFloatTy()) + targetValue = CreateBitCast(targetValue, getInt32Ty()); + else if (targetValue->getType() == getDoubleTy()) + targetValue = CreateBitCast(targetValue, getInt64Ty()); + else + assert(targetValue->getType()->isIntegerTy()); + + // Handle Loop Condition Block + // remove the default br generated by splitBlockBefore. + loopBlock->getTerminator()->eraseFromParent(); + SetInsertPoint(loopBlock); + Value *laneValue = CreateSubgroupBroadcastFirst(targetValue); + Value *isEqual = CreateICmpEQ(laneValue, targetValue); + Value *mask = createGroupBallot(isEqual); + CreateCondBr(isEqual, loopEndBlock, loopBlock); + + // Handle Loop End + SetInsertPoint(&*loopEndBlock->begin()); + // Match expects a <4 x i32> return, so we need to turn the i64 into that. + Value *finalResult = CreateBitCast(mask, FixedVectorType::get(getInt32Ty(), 2)); + + ElementCount elementCount = cast(finalResult->getType())->getElementCount(); + return CreateShuffleVector(finalResult, ConstantVector::getSplat(elementCount, getInt32(0)), + ArrayRef{0, 1, 2, 3}); +} + // ===================================================================================================================== // Create The group arithmetic operation identity. // @@ -1276,8 +1425,17 @@ Value *BuilderImpl::createGroupBallot(Value *const value) { // Check the type is definitely an boolean. assert(value->getType()->isIntegerTy(1)); + Value *result = value; + + // For waveOpsExcludeHelperLanes mode, we need mask away the helperlane. + const auto &fragmentMode = m_pipelineState->getShaderModes()->getFragmentShaderMode(); + if (m_shaderStage == ShaderStageFragment && fragmentMode.waveOpsExcludeHelperLanes) { + auto isLive = CreateIntrinsic(Intrinsic::amdgcn_live_mask, {}, {}, nullptr, {}); + result = CreateAnd(isLive, result); + } + unsigned waveSize = getShaderWaveSize(); - Value *result = CreateIntrinsic(getIntNTy(waveSize), Intrinsic::amdgcn_ballot, value); + result = CreateIntrinsic(getIntNTy(waveSize), Intrinsic::amdgcn_ballot, result); // If we have a 32-bit subgroup size, we need to turn the 32-bit ballot result into a 64-bit result. if (waveSize <= 32) @@ -1285,3 +1443,18 @@ Value *BuilderImpl::createGroupBallot(Value *const value) { return result; } + +// ===================================================================================================================== +// Search the MSB index of the mask, not handle zero. +// +// @param mask : The lane mask. +Value *BuilderImpl::createFindMsb(Value *const mask) { + // counts the number of leading zeros. + Value *result = CreateBinaryIntrinsic(Intrinsic::ctlz, mask, getTrue()); + + if (getShaderSubgroupSize() == 64) + result = CreateTrunc(result, getInt32Ty()); + + // reverse the count from the bottom. + return CreateSub(getInt32((getShaderSubgroupSize() == 64) ? 63 : 31), result); +} diff --git a/lgc/builder/YCbCrConverter.cpp b/lgc/builder/YCbCrConverter.cpp index b8efbe2332..bd5fa54d6b 100644 --- a/lgc/builder/YCbCrConverter.cpp +++ b/lgc/builder/YCbCrConverter.cpp @@ -819,7 +819,7 @@ Value *YCbCrConverter::convertColor(Type *resultTy, SamplerYCbCrModelConversion case SamplerYCbCrModelConversion::YCbCr601: case SamplerYCbCrModelConversion::YCbCr709: case SamplerYCbCrModelConversion::YCbCr2020: { - // inputVec = RangeExpaned(C'_rgba) + // inputVec = RangeExpand(C'_rgba) Value *inputVec = m_builder->CreateFClamp(rangeExpand(range, channelBits, subImage), minVec, maxVec); Value *inputCr = m_builder->CreateExtractElement(inputVec, m_builder->getInt64(0)); diff --git a/lgc/elfLinker/ColorExportShader.cpp b/lgc/elfLinker/ColorExportShader.cpp index 6817ece696..0b65c9dcfe 100644 --- a/lgc/elfLinker/ColorExportShader.cpp +++ b/lgc/elfLinker/ColorExportShader.cpp @@ -112,21 +112,43 @@ Module *ColorExportShader::generate() { BuilderBase builder(ret); SmallVector values(MaxColorTargets + 1, nullptr); - for (unsigned idx = 0; idx != m_exports.size(); ++idx) { + unsigned exportSize = m_exports.size(); + unsigned lastIndex = 0; + for (unsigned idx = 0; idx != exportSize; ++idx) { values[m_exports[idx].hwColorTarget] = colorExportFunc->getArg(idx); + ++lastIndex; } PalMetadata palMetadata{m_pipelineState, m_pipelineState->useRegisterFieldFormat()}; bool dummyExport = m_lgcContext->getTargetInfo().getGfxIpVersion().major < 10 || m_killEnabled; - fragColorExport.generateExportInstructions(m_exports, values, dummyExport, &palMetadata, builder); + Value *dynamicIsDualSource = colorExportFunc->getArg(lastIndex); + fragColorExport.generateExportInstructions(m_exports, values, dummyExport, &palMetadata, builder, + dynamicIsDualSource); + // Handle on the dualSourceBlend case which may have two blocks with two returnInsts + SmallVector retInsts; if (m_pipelineState->getOptions().enableColorExportShader) { + for (auto &block : llvm::reverse(*colorExportFunc)) { + if (auto ret = dyn_cast(block.getTerminator())) { + retInsts.push_back(ret); + } + } + } + for (ReturnInst *inst : llvm::reverse(retInsts)) { + builder.SetInsertPoint(inst); builder.CreateIntrinsic(Intrinsic::amdgcn_endpgm, {}, {}); builder.CreateUnreachable(); - ret->eraseFromParent(); + inst->eraseFromParent(); } + // Set pipeline hash. + auto internalPipelineHash = + palMetadata.getPipelineNode()[Util::Abi::PipelineMetadataKey::InternalPipelineHash].getArray(true); + const auto &options = m_pipelineState->getOptions(); + internalPipelineHash[0] = options.hash[0]; + internalPipelineHash[1] = options.hash[1]; + palMetadata.updateDbShaderControl(); palMetadata.record(colorExportFunc->getParent()); @@ -147,6 +169,7 @@ Function *ColorExportShader::createColorExportFunc() { SmallVector entryTys; for (const auto &exp : m_exports) entryTys.push_back(exp.ty); + entryTys.push_back(Type::getInt32Ty(getContext())); auto funcTy = FunctionType::get(Type::getVoidTy(getContext()), entryTys, false); // Create the function. Mark SGPR inputs as "inreg". @@ -156,6 +179,9 @@ Function *ColorExportShader::createColorExportFunc() { else func->setCallingConv(CallingConv::AMDGPU_PS); + unsigned inRegIndex = m_exports.size(); + func->addParamAttr(inRegIndex, Attribute::InReg); + func->setDLLStorageClass(GlobalValue::DLLExportStorageClass); setShaderStage(func, ShaderStageFragment); diff --git a/lgc/elfLinker/ElfLinker.cpp b/lgc/elfLinker/ElfLinker.cpp index efb6f34b76..7182297955 100644 --- a/lgc/elfLinker/ElfLinker.cpp +++ b/lgc/elfLinker/ElfLinker.cpp @@ -370,14 +370,6 @@ StringRef ElfLinkerImpl::buildColorExportShader(ArrayRef export if (enableKill) copyColorShader->enableKill(); - PalMetadata *metadata = m_pipelineState->getPalMetadata(); - // Set pipeline hash. - auto internalPipelineHash = - metadata->getPipelineNode()[Util::Abi::PipelineMetadataKey::InternalPipelineHash].getArray(true); - const auto &options = m_pipelineState->getOptions(); - internalPipelineHash[0] = options.hash[0]; - internalPipelineHash[1] = options.hash[1]; - return copyColorShader->getElfBlob(); } diff --git a/lgc/include/lgc/builder/BuilderImpl.h b/lgc/include/lgc/builder/BuilderImpl.h index 37357e07a4..d401b4a123 100644 --- a/lgc/include/lgc/builder/BuilderImpl.h +++ b/lgc/include/lgc/builder/BuilderImpl.h @@ -728,6 +728,10 @@ class BuilderImpl : public BuilderDefs { llvm::Value *CreateSubgroupClusteredExclusive(GroupArithOp groupArithOp, llvm::Value *const value, llvm::Value *const clusterSize, const llvm::Twine &instName = ""); + // Create a subgroup clustered multi exclusive scan. + llvm::Value *CreateSubgroupClusteredMultiExclusive(GroupArithOp groupArithOp, llvm::Value *const value, + llvm::Value *const mask, const llvm::Twine &instName = ""); + // Create a quad broadcast. llvm::Value *CreateSubgroupQuadBroadcast(llvm::Value *const value, llvm::Value *const index, bool inWQM = true, const llvm::Twine &instName = ""); @@ -756,6 +760,9 @@ class BuilderImpl : public BuilderDefs { // Create a subgroup mbcnt. llvm::Value *CreateSubgroupMbcnt(llvm::Value *const mask, const llvm::Twine &instName = ""); + // Create a subgroup partition. + llvm::Value *CreateSubgroupPartition(llvm::Value *const value, const llvm::Twine &instName = ""); + private: unsigned getShaderSubgroupSize(); unsigned getShaderWaveSize(); @@ -781,6 +788,7 @@ class BuilderImpl : public BuilderDefs { uint16_t getDsSwizzleBitMode(uint8_t xorMask, uint8_t orMask, uint8_t andMask); uint16_t getDsSwizzleQuadMode(uint8_t lane0, uint8_t lane1, uint8_t lane2, uint8_t lane3); llvm::Value *createGroupBallot(llvm::Value *const value); + llvm::Value *createFindMsb(llvm::Value *const mask); }; } // namespace lgc diff --git a/lgc/include/lgc/patch/FragColorExport.h b/lgc/include/lgc/patch/FragColorExport.h index 4b4640dab4..6485b9f58b 100644 --- a/lgc/include/lgc/patch/FragColorExport.h +++ b/lgc/include/lgc/patch/FragColorExport.h @@ -62,7 +62,8 @@ class FragColorExport { FragColorExport(llvm::LLVMContext *context, PipelineState *pipelineState); void generateExportInstructions(llvm::ArrayRef info, llvm::ArrayRef values, - bool dummyExport, PalMetadata *palMetadata, BuilderBase &builder); + bool dummyExport, PalMetadata *palMetadata, BuilderBase &builder, + llvm::Value *dynamicIsDualSource); static void setDoneFlag(llvm::Value *exportInst, BuilderBase &builder); static llvm::CallInst *addDummyExport(BuilderBase &builder); static llvm::Function *generateNullFragmentShader(llvm::Module &module, PipelineState *pipelineState, @@ -75,10 +76,11 @@ class FragColorExport { FragColorExport() = delete; FragColorExport(const FragColorExport &) = delete; FragColorExport &operator=(const FragColorExport &) = delete; - + void updateColorExportInfoWithBroadCastInfo(llvm::ArrayRef originExpinfo, + llvm::SmallVector &outExpinfo, unsigned *pCbShaderMask); llvm::Value *handleColorExportInstructions(llvm::Value *output, unsigned int hwColorExport, BuilderBase &builder, - ExportFormat expFmt, const bool signedness, unsigned channelWriteMask); - + ExportFormat expFmt, const bool signedness, unsigned channelWriteMask, + const bool isDualSourceBlend); llvm::Value *convertToHalf(llvm::Value *value, bool signedness, BuilderBase &builder) const; llvm::Value *convertToFloat(llvm::Value *value, bool signedness, BuilderBase &builder) const; llvm::Value *convertToInt(llvm::Value *value, bool signedness, BuilderBase &builder) const; @@ -117,7 +119,7 @@ class LowerFragColorExport : public llvm::PassInfoMixin { void collectExportInfoForGenericOutputs(llvm::Function *fragEntryPoint, BuilderBase &builder); void collectExportInfoForBuiltinOutput(llvm::Function *module, BuilderBase &builder); llvm::Value *generateValueForOutput(llvm::Value *value, llvm::Type *outputTy, BuilderBase &builder); - void createTailJump(llvm::Function *fragEntryPoint, BuilderBase &builder); + void createTailJump(llvm::Function *fragEntryPoint, BuilderBase &builder, llvm::Value *isDualSource); llvm::LLVMContext *m_context; // The context the pass is being run in. PipelineState *m_pipelineState; // The pipeline state diff --git a/lgc/include/lgc/patch/LowerCooperativeMatrix.h b/lgc/include/lgc/patch/LowerCooperativeMatrix.h index f472dd68d7..6e3b9d4bab 100644 --- a/lgc/include/lgc/patch/LowerCooperativeMatrix.h +++ b/lgc/include/lgc/patch/LowerCooperativeMatrix.h @@ -194,11 +194,16 @@ class LowerCooperativeMatrix : public Patch, public llvm::PassInfoMixin expFormats); // Updates the CB shader mask information that depends on the exports. - void updateCbShaderMask(llvm::ArrayRef exps); + void updateCbShaderMask(unsigned cbShaderMask); // Updates the DB shader control that depends on the CB state. void updateDbShaderControl(); diff --git a/lgc/include/lgc/state/PipelineState.h b/lgc/include/lgc/state/PipelineState.h index ed2ce0d118..2e73720883 100644 --- a/lgc/include/lgc/state/PipelineState.h +++ b/lgc/include/lgc/state/PipelineState.h @@ -176,7 +176,7 @@ class PipelineState final : public Pipeline { // Compute the ExportFormat (as an opaque int) of the specified color export location with the specified output // type. Only the number of elements of the type is significant. - unsigned computeExportFormat(llvm::Type *outputTy, unsigned location) override final; + unsigned computeExportFormat(llvm::Type *outputTy, unsigned location, bool isDynamicDsBlend = false) override final; // Set entire pipeline state from metadata in an IR module. This is used by the lgc command-line utility // for its link option. @@ -284,9 +284,9 @@ class PipelineState final : public Pipeline { const VertexInputDescription *findVertexInputDescription(unsigned location) const; // Accessors for color export state - const ColorExportFormat &getColorExportFormat(unsigned location); + const ColorExportFormat &getColorExportFormat(unsigned location, bool isDynamicDsBlend = false); const bool hasColorExportFormats() { return !m_colorExportFormats.empty(); } - ColorExportState &getColorExportState() { return m_colorExportState; } + const ColorExportState &getColorExportState() { return m_colorExportState; } // Accessors for pipeline state unsigned getDeviceIndex() const { return m_deviceIndex; } diff --git a/lgc/include/lgc/state/ResourceUsage.h b/lgc/include/lgc/state/ResourceUsage.h index 7522eeb0c7..268641b2b2 100644 --- a/lgc/include/lgc/state/ResourceUsage.h +++ b/lgc/include/lgc/state/ResourceUsage.h @@ -592,9 +592,10 @@ struct InterfaceData { // Fragment shader struct { - unsigned viewId; // View ID - unsigned primMask; // Primitive mask - unsigned sampleInfo; // Sample Info: numSample + samplePattern + unsigned viewId; // View ID + unsigned primMask; // Primitive mask + unsigned sampleInfo; // Sample Info: numSample + samplePattern + unsigned dynamicDualSrcBlendInfo; // dualSrcBlendDynamicValue // Perspective interpolation (I/J) struct { diff --git a/lgc/interface/lgc/Builder.h b/lgc/interface/lgc/Builder.h index 04561228ad..d2e86991d8 100644 --- a/lgc/interface/lgc/Builder.h +++ b/lgc/interface/lgc/Builder.h @@ -110,11 +110,6 @@ class InOutInfo { m_data.bits.component = component; } - bool isDualSourceBlendDynamic() const { return m_data.bits.dualSourceBlendDynamic; } - void setDualSourceBlendDynamic(bool dualSourceBlendDynamic = true) { - m_data.bits.dualSourceBlendDynamic = dualSourceBlendDynamic; - } - private: union { struct { @@ -130,7 +125,6 @@ class InOutInfo { // whole array or of an element with a variable index. unsigned perPrimitive : 1; // Mesh shader output: whether it is a per-primitive output unsigned component : 2; // Component offset, specifying which components within a location is consumed - unsigned dualSourceBlendDynamic : 1; // Fs output: whether it's dynamic dual source blend output } bits; unsigned u32All; } m_data; @@ -1581,6 +1575,15 @@ class Builder : public BuilderDefs { llvm::Value *CreateSubgroupClusteredExclusive(GroupArithOp groupArithOp, llvm::Value *const value, llvm::Value *const clusterSize, const llvm::Twine &instName = ""); + // Create a subgroup clustered multi exclusive scan. + // + // @param groupArithOp : The group arithmetic operation to perform + // @param value : The value to perform on + // @param mask : The mask of cluster. Must be <4 x i32> type. + // @param instName : Name to give instruction(s) + llvm::Value *CreateSubgroupClusteredMultiExclusive(GroupArithOp groupArithOp, llvm::Value *const value, + llvm::Value *const mask, const llvm::Twine &instName = ""); + // Create a quad broadcast. // // @param value : The value to broadcast @@ -1639,6 +1642,12 @@ class Builder : public BuilderDefs { // @param instName : Name to give instruction(s) llvm::Value *CreateSubgroupMbcnt(llvm::Value *const mask, const llvm::Twine &instName = ""); + // Create a subgroup partition. + // + // @param value : The value to contribute + // @param instName : Name to give instruction(s) + llvm::Value *CreateSubgroupPartition(llvm::Value *const value, const llvm::Twine &instName = ""); + private: Builder() = delete; Builder(const Builder &) = delete; diff --git a/lgc/interface/lgc/BuilderCommon.h b/lgc/interface/lgc/BuilderCommon.h index f006768365..ff57b1001f 100644 --- a/lgc/interface/lgc/BuilderCommon.h +++ b/lgc/interface/lgc/BuilderCommon.h @@ -95,12 +95,13 @@ class BuilderCommon : public llvm_dialects::Builder { }; enum CooperativeMatrixElementType { - Unknown = 0, // Unknown - Float16, // 16-bit floating-point - Float32, // 32-bit floating-point - Int8, // 8-bit integer - Int16, // 16-bit integer - Int32 // 32 bit integer + Unknown = 0, // Unknown + Float16, // 16-bit floating-point + Float32, // 32-bit floating-point + Int8, // 8-bit integer + Int16, // 16-bit integer + Int32, // 32 bit integer + Float16Packed, // packed 16-bit floating-point }; // Layout is virtual concept, eg: 16bit and 32bit for matrixC will share the same layout initially. @@ -204,7 +205,8 @@ class BuilderCommon : public llvm_dialects::Builder { // Create cooperative MatrixTimesScalar binary operation // // @param matrix : It should be cooperative matrix. - // @param scalar : It should be scalar type. + // @param scalar : It should be scalar type. If the matrix is a packed + // accumulator matrix, the scalar has to be a <2 x half> vector. // @param elemType : Name to give instruction(s). // @param layout : Identify A/B matrices or C/D matrices. llvm::Value *CreateCoopMatrixTimesScalar(llvm::Value *matrix, llvm::Value *scalar, @@ -226,14 +228,34 @@ class BuilderCommon : public llvm_dialects::Builder { // @param coopMatrixc : Accumulator cooperative matrix. // @param isSignedA : Identify the signess for matrix A's element type // @param isSignedB : Identify the signess for matrix B's element type - // @param isSat : SaturatingAccumulation for calculation + // @param isSatOrOpsel : SaturatingAccumulation for calculation. In the + // case of 16-bit floating point matrices, this bit acts as an opsel bit. If + // it is set to false, we store the result in the lower half of the registers. + // If it is true, we store it in the upper half. + // @param isTied : If true, the output matrix has to be the same as the + // input accumulator (i.e., D has to be C) // @param accumElemType : The component type of the matrix c // @param factorElemType : The component type of the matrix a llvm::Value *CreateCooperativeMatrixMulAdd(llvm::Value *coopMatrixa, llvm::Value *coopMatrixb, - llvm::Value *coopMatrixc, bool isSignedA, bool isSignedB, bool isSat, - CooperativeMatrixElementType accumElemType, + llvm::Value *coopMatrixc, bool isSignedA, bool isSignedB, + bool isSatOrOpsel, bool isTied, CooperativeMatrixElementType accumElemType, CooperativeMatrixElementType factorElemType, const llvm::Twine &instName = ""); + + // ===================================================================================================================== + // Create cooperative matrix pack operation + // + // @param matrixCLo : Lower Accumulator cooperative matrix. + // @param matrixCHi : Upper Accumulator cooperative matrix. + llvm::Value *CreateCooperativeMatrixPack(llvm::Value *matrixCLo, llvm::Value *matrixCHi, + const llvm::Twine &instName = ""); + + // ===================================================================================================================== + // Create cooperative matrix unpack operation + // + // @param packedMatrix : Packed Accumulator cooperative matrices. + // @param high: Whether to get the matrix stored in the upper half of the registers. + llvm::Value *CreateCooperativeMatrixUnpack(llvm::Value *packedMatrix, bool high, const llvm::Twine &instName = ""); }; } // namespace lgc diff --git a/lgc/interface/lgc/Pipeline.h b/lgc/interface/lgc/Pipeline.h index 5d8a3036ea..e39c3609ad 100644 --- a/lgc/interface/lgc/Pipeline.h +++ b/lgc/interface/lgc/Pipeline.h @@ -473,7 +473,7 @@ struct ColorExportFormat { struct ColorExportState { unsigned alphaToCoverageEnable; // Enable alpha to coverage unsigned dualSourceBlendEnable; // Blend state bound at draw time will use a dual source blend mode - unsigned dynamicDualSourceBlendEnable; // Dynamic dual source blend enable + unsigned dualSourceBlendDynamicEnable; // Dual source blend mode is dynamically set }; // MultiView supporting mode @@ -661,6 +661,7 @@ struct FragmentShaderMode { unsigned postDepthCoverage; unsigned earlyAndLatFragmentTests; unsigned innerCoverage; + unsigned waveOpsExcludeHelperLanes; ConservativeDepth conservativeDepth; ConservativeDepth conservativeStencilFront; ConservativeDepth conservativeStencilBack; @@ -954,7 +955,7 @@ class Pipeline { // Compute the ExportFormat (as an opaque int) of the specified color export location with the specified output // type. Only the number of elements of the type is significant. // This is not used in a normal compile; it is only used by amdllpc's -check-auto-layout-compatible option. - virtual unsigned computeExportFormat(llvm::Type *outputTy, unsigned location) = 0; + virtual unsigned computeExportFormat(llvm::Type *outputTy, unsigned location, bool isDynamicDsBlend = false) = 0; private: LgcContext *m_builderContext; // Builder context diff --git a/lgc/patch/CombineCooperativeMatrix.cpp b/lgc/patch/CombineCooperativeMatrix.cpp index fbf3588677..91aa4acdc3 100644 --- a/lgc/patch/CombineCooperativeMatrix.cpp +++ b/lgc/patch/CombineCooperativeMatrix.cpp @@ -35,6 +35,9 @@ #include "lgc/patch/CombineCooperativeMatrix.h" #include "lgc/Builder.h" #include "lgc/state/Defs.h" +#include "lgc/state/PipelineState.h" +#include "lgc/state/TargetInfo.h" +#include "llvm/ADT/SmallVector.h" #include #define DEBUG_TYPE "lgc-combine-cooperative-matrix" @@ -59,14 +62,20 @@ struct Shape { // networks of phi nodes. struct DataFlowComponent { SmallVector inputs; - SmallVector phis; + // need to track in- and outputs of these nodes + struct { + SmallVector phis; + SmallVector timesScalars; + SmallVector binOps; + } inner; SmallVector outputs; std::optional shape; }; class CooperativeMatrixCombiner { public: - CooperativeMatrixCombiner(Function &function) : m_function(function), b(function.getContext()) {} + CooperativeMatrixCombiner(Function &function, GfxIpVersion gfxIpVersion) + : m_function(function), b(function.getContext()), m_gfxIpVersion(gfxIpVersion) {} bool run(); @@ -75,9 +84,13 @@ class CooperativeMatrixCombiner { void foldTo(Value *from, Value *to); bool tryFold(CallInst *op); bool tryFoldComponentContaining(Value *start); + Instruction *findFirstUser(Instruction *instruction); + Value *tryFoldTimesScalar(CallInst *timesScalarLo, CallInst *timesScalarHi, Value *packedMatrix); + bool tryFoldMuladd(SmallVector muladds); Function &m_function; BuilderCommon b; + GfxIpVersion m_gfxIpVersion; std::vector m_eraseList; }; @@ -92,8 +105,9 @@ bool CooperativeMatrixCombiner::run() { bool changed = false; - // Step 1: Collect transposes and converts + // Step 1: Collect transposes, converts and muladds std::vector ops; + MapVector> muladds; for (Function &fn : m_function.getParent()->functions()) { if (!fn.isDeclaration()) @@ -113,6 +127,20 @@ bool CooperativeMatrixCombiner::run() { ops.push_back(call); } } +#if !defined(LLVM_MAIN_REVISION) || LLVM_MAIN_REVISION >= 479080 + // wmma packing on gfx11 only possible with new wmma_f16_tied intrinsic + } else if (m_gfxIpVersion.major == 11 && fn.getName().startswith(lgcName::CooperativeMatrixMulAdd)) { + for (User *user : fn.users()) { + if (auto *call = dyn_cast(user)) { + Builder::CooperativeMatrixElementType accumElemType = static_cast( + cast(call->getOperand(7))->getZExtValue()); + bool isPackable = accumElemType == Builder::CooperativeMatrixElementType::Float16; + if (call->getFunction() == &m_function && isPackable) { + muladds[call->getParent()].push_back(call); + } + } + } +#endif } } @@ -132,6 +160,21 @@ bool CooperativeMatrixCombiner::run() { m_eraseList.clear(); } } +#if !defined(LLVM_MAIN_REVISION) || LLVM_MAIN_REVISION >= 479080 + // wmma packing on gfx11 only possible with new wmma_f16_tied intrinsic + for (auto muladdsPerBB : muladds) { + changed |= tryFoldMuladd(std::move(muladdsPerBB.second)); + + for (Instruction *inst : llvm::reverse(m_eraseList)) { + if (inst->use_empty()) + inst->eraseFromParent(); + } + m_eraseList.clear(); + } + + muladds.clear(); +#endif + ops.clear(); return changed; @@ -162,8 +205,9 @@ Shape CooperativeMatrixCombiner::getShapeOfTranspose(CallInst *transpose) { void CooperativeMatrixCombiner::foldTo(Value *from, Value *to) { from->replaceAllUsesWith(to); - if (auto *fromInst = dyn_cast(from)) + if (auto *fromInst = dyn_cast(from)) { m_eraseList.push_back(fromInst); + } } // ===================================================================================================================== @@ -236,48 +280,74 @@ bool CooperativeMatrixCombiner::tryFoldComponentContaining(Value *start) { // Step 1: Discover the component DataFlowComponent component; - SmallVector worklist; + SmallVector worklistForward; + SmallVector worklistBackward; + + auto foundInner = [&](Value *val) { + if (auto *phi = dyn_cast(val)) { + if (is_contained(component.inner.phis, phi)) + return true; - if (auto *phi = dyn_cast(start)) - component.phis.push_back(phi); - else + component.inner.phis.push_back(phi); + worklistForward.push_back(phi); + for (Value *incoming : phi->incoming_values()) { + worklistBackward.push_back(incoming); + } + return true; + } + if (auto *call = dyn_cast(val)) { + if (auto *callee = call->getCalledFunction()) { + if (callee->getName().starts_with(lgcName::CooperativeMatrixTimesScalar)) { + if (is_contained(component.inner.timesScalars, call)) + return true; + + component.inner.timesScalars.push_back(call); + worklistForward.push_back(call); + worklistBackward.push_back(call->getArgOperand(0)); + return true; + } + if (callee->getName().starts_with(lgcName::CooperativeMatrixBinOp)) { + if (is_contained(component.inner.binOps, call)) + return true; + + component.inner.binOps.push_back(call); + worklistForward.push_back(call); + worklistBackward.push_back(call->getArgOperand(1)); + worklistBackward.push_back(call->getArgOperand(2)); + return true; + } + return false; + } + } + return false; + }; + + if (!foundInner(start)) { component.inputs.push_back(start); - worklist.push_back(start); + worklistForward.push_back(start); + } do { - Value *current = worklist.pop_back_val(); - - auto foundPhi = [&](PHINode *phi) { - if (llvm::any_of(component.phis, [=](auto elem) { return elem == phi; })) - return; - component.phis.push_back(phi); - worklist.push_back(phi); - }; + Value *current = worklistForward.pop_back_val(); for (Use &use : current->uses()) { - if (auto *phi = dyn_cast(use.getUser())) { - foundPhi(phi); - continue; + if (!foundInner(use.getUser())) { + component.outputs.push_back(&use); } - - component.outputs.push_back(&use); } - if (auto *phi = dyn_cast(current)) { - for (Value *incoming : phi->incoming_values()) { - if (auto *parentPhi = dyn_cast(incoming)) { - foundPhi(parentPhi); - } else { - if (llvm::any_of(component.inputs, [=](auto elem) { return elem == incoming; })) - continue; - if (!isa(incoming)) { - component.inputs.push_back(incoming); - worklist.push_back(incoming); - } - } + while (!worklistBackward.empty()) { + Value *incoming = worklistBackward.pop_back_val(); + if (is_contained(component.inputs, incoming)) + continue; + if (foundInner(incoming)) + continue; + if (!isa(incoming)) { + component.inputs.push_back(incoming); + worklistForward.push_back(incoming); } } - } while (!worklist.empty()); + } while (!worklistForward.empty()); // Step 2: Analyze the inputs and outputs. std::optional otherLayout; @@ -452,7 +522,7 @@ bool CooperativeMatrixCombiner::tryFoldComponentContaining(Value *start) { // Handle generic outputs that need to be transposed explicitly. Value *&transposed = outTransposed[use->get()]; if (!transposed) { - if (auto *phi = cast(use->get())) { + if (auto *phi = dyn_cast(use->get())) { b.SetInsertPoint(phi->getParent(), phi->getParent()->getFirstInsertionPt()); } else { auto *def = cast(use->get()); @@ -479,9 +549,9 @@ bool CooperativeMatrixCombiner::tryFoldComponentContaining(Value *start) { // Cache for newly inserted relayout convert operations. DenseMap outRelayouted; - // Force-override phi types if necessary - if (!component.phis.empty() && component.phis[0]->getType() != otherType) { - for (PHINode *phi : component.phis) { + // Force-override inner nodes if necessary + if (!component.inner.phis.empty() && component.inner.phis[0]->getType() != otherType) { + for (PHINode *phi : component.inner.phis) { phi->mutateType(otherType); for (Use &use : phi->incoming_values()) { @@ -501,6 +571,18 @@ bool CooperativeMatrixCombiner::tryFoldComponentContaining(Value *start) { } } + for (CallInst *timesScalar : component.inner.timesScalars) { + timesScalar->mutateType(otherType); + timesScalar->setArgOperand(3, b.getInt32((unsigned)*otherLayout)); + continue; + } + + for (CallInst *binOp : component.inner.binOps) { + binOp->mutateType(otherType); + binOp->setArgOperand(4, b.getInt32((unsigned)*otherLayout)); + continue; + } + for (Value *input : component.inputs) { // Handle inputs for which the relayout can be folded or absorbed. if (auto *call = dyn_cast(input)) { @@ -577,7 +659,7 @@ bool CooperativeMatrixCombiner::tryFoldComponentContaining(Value *start) { // Handle generic outputs that need a new convert operation inserted. Value *&relayouted = outRelayouted[use->get()]; if (!relayouted) { - if (auto *phi = cast(use->get())) { + if (auto *phi = dyn_cast(use->get())) { b.SetInsertPoint(phi->getParent(), phi->getParent()->getFirstInsertionPt()); } else { auto *def = cast(use->get()); @@ -598,6 +680,251 @@ bool CooperativeMatrixCombiner::tryFoldComponentContaining(Value *start) { return false; } +Instruction *CooperativeMatrixCombiner::findFirstUser(Instruction *instruction) { + Instruction *earliestUser = nullptr; + for (auto *user : instruction->users()) { + auto *userInst = dyn_cast(user); + // We only pack instructions inside the same basic block. + // Therefore, users outside the BB don't interfere + if (instruction->getParent() != userInst->getParent()) + continue; + + if (dyn_cast(userInst)) + continue; + + if (!earliestUser || userInst->comesBefore(earliestUser)) + earliestUser = userInst; + } + return earliestUser; +} + +bool CooperativeMatrixCombiner::tryFoldMuladd(SmallVector muladds) { + bool changed = false; + + auto cmp = [](CallInst *a, CallInst *b) { return b->comesBefore(a); }; + stable_sort(muladds, cmp); + + do { + auto *muladdLo = muladds.pop_back_val(); + auto *packInsertPoint = cast(muladdLo); + + struct PackingComponents { + Value *matrixLo; + Value *matrixHi; + Value *packedAccum; + }; + SmallVector worklist; + SmallVector> unpackedUses; + SmallVector muladdChain; + + auto *matCLo = muladdLo->getArgOperand(2); + + muladdChain.push_back(muladdLo); + muladdLo->setArgOperand(5, b.getInt1(false)); + while (muladdLo->hasOneUse()) { + auto *next = dyn_cast(*muladdLo->users().begin()); + + if (!is_contained(muladds, next)) + break; + + next->setArgOperand(5, b.getInt1(false)); + muladdChain.push_back(next); + muladdLo = next; +#if LLVM_MAIN_REVISION && LLVM_MAIN_REVISION < 478769 + llvm::erase_value(muladds, muladdLo); +#else + llvm::erase(muladds, muladdLo); +#endif + } + + Instruction *firstLoUser = findFirstUser(muladdLo); + + CallInst *muladdHi = nullptr; + for (auto *candidate : llvm::reverse(muladds)) { + if (firstLoUser && firstLoUser->comesBefore(candidate)) + continue; + + if (auto *matCHi = dyn_cast(candidate->getArgOperand(2))) { + if (matCHi->getParent() == muladdLo->getParent() && packInsertPoint->comesBefore(matCHi)) { + continue; + } + } + + muladdHi = candidate; + break; + } + + if (!muladdHi) + continue; + + auto *matCHi = muladdHi->getArgOperand(2); + + muladdChain.push_back(muladdHi); + muladdHi->setArgOperand(5, b.getInt1(true)); +#if LLVM_MAIN_REVISION && LLVM_MAIN_REVISION < 478769 + llvm::erase_value(muladds, muladdLo); +#else + llvm::erase(muladds, muladdHi); +#endif + while (muladdHi->hasOneUse()) { + auto *next = dyn_cast(*muladdHi->users().begin()); + if (!is_contained(muladds, next)) { + break; + } + + if (firstLoUser && firstLoUser->comesBefore(next)) { + break; + } + next->setArgOperand(5, b.getInt1(true)); + muladdChain.push_back(next); + muladdHi = next; +#if LLVM_MAIN_REVISION && LLVM_MAIN_REVISION < 478769 + llvm::erase_value(muladds, muladdLo); +#else + llvm::erase(muladds, next); +#endif + } + + auto cmp = [&](CallInst *a, CallInst *b) { return a->comesBefore(b); }; + stable_sort(muladdChain, cmp); + + // if we have a loop, then the accumulator matrices come from the phi nodes. + // in that case, we need to pack in the predecessor blocks, so all the + // incoming values are packed accumulators. + PHINode *const phiLo = dyn_cast(matCLo); + PHINode *const phiHi = dyn_cast(matCHi); + Value *curAccum = nullptr; + if (phiLo && phiHi && phiLo->getParent() == phiHi->getParent()) { + for (BasicBlock *incoming : phiLo->blocks()) { + b.SetInsertPoint(incoming->getTerminator()); + auto *matCLo = phiLo->getIncomingValueForBlock(incoming); + auto *matCHi = phiHi->getIncomingValueForBlock(incoming); + auto *packed = b.CreateCooperativeMatrixPack(matCLo, matCHi); + phiLo->setIncomingValueForBlock(incoming, packed); + phiHi->setIncomingValueForBlock(incoming, packed); + } + curAccum = phiHi; + worklist.push_back({phiLo, phiHi, phiHi}); + } else { + // otherwise, we pack just before the first muladd + b.SetInsertPoint(packInsertPoint); + curAccum = b.CreateCooperativeMatrixPack(matCLo, matCHi); + } + + for (auto *next : muladdChain) { + next->setArgOperand(2, curAccum); + next->setArgOperand(6, b.getInt1(true)); + curAccum = next; + } + + worklist.push_back({muladdLo, muladdHi, curAccum}); + while (!worklist.empty()) { + auto current = worklist.pop_back_val(); + + for (Use &use : current.matrixLo->uses()) { + if (is_contained(muladdChain, use.getUser())) + continue; + + unpackedUses.push_back({use, false}); + } + for (Use &use : current.matrixHi->uses()) { + if (is_contained(muladdChain, use.getUser())) + continue; + + if (auto *call = dyn_cast(use.getUser())) { + if (auto *callee = call->getCalledFunction()) { + if (callee->getName().starts_with(lgcName::CooperativeMatrixTimesScalar)) { + auto *candidate = llvm::find_if(unpackedUses, [&](auto pair) { + if (auto *call = dyn_cast(pair.first.getUser())) { + if (auto *callee = call->getCalledFunction()) { + if (callee->getName().startswith(lgcName::CooperativeMatrixTimesScalar) && + call->getArgOperand(0) == current.matrixLo) { + return true; + } + } + } + return false; + }); + + if (candidate == unpackedUses.end()) { + unpackedUses.push_back({use, true}); + continue; + } + + auto *timesScalarLo = cast(candidate->first.getUser()); + auto *timesScalarHi = call; + auto *timesScalarPacked = tryFoldTimesScalar(timesScalarLo, timesScalarHi, current.packedAccum); + + if (timesScalarPacked) { + worklist.push_back({timesScalarLo, timesScalarHi, timesScalarPacked}); + continue; + } + } + } + } + + unpackedUses.push_back({use, true}); + } + + for (auto use : unpackedUses) { + if (is_contained(m_eraseList, use.first.getUser())) + continue; + + if (auto *call = dyn_cast(use.first.getUser())) { + if (call->getCalledFunction()->getName().startswith(lgcName::CooperativeMatrixPack) && + call->getArgOperand(0) == current.matrixLo && call->getArgOperand(1) == current.matrixHi) { + foldTo(call, current.packedAccum); + continue; + } + } + + if (auto *phi = dyn_cast(use.first.getUser())) { + auto *predecessor = phi->getIncomingBlock(use.first); + b.SetInsertPoint(predecessor->getTerminator()); + } else { + b.SetInsertPoint(cast(use.first.getUser())); + } + auto unpacked = b.CreateCooperativeMatrixUnpack(current.packedAccum, use.second); + use.first.set(unpacked); + } + unpackedUses.clear(); + } + if (phiLo && phiHi) + foldTo(phiLo, phiHi); + + changed = true; + } while (!muladds.empty()); + + return changed; +} + +Value *CooperativeMatrixCombiner::tryFoldTimesScalar(CallInst *timesScalarLo, CallInst *timesScalarHi, + Value *packedMatrix) { + if (timesScalarLo->getParent() != timesScalarHi->getParent()) { + return nullptr; + } + + auto *earlierInst = timesScalarLo->comesBefore(timesScalarHi) ? timesScalarLo : timesScalarHi; + auto *laterInst = earlierInst == timesScalarLo ? timesScalarHi : timesScalarLo; + auto *earliestUser = findFirstUser(earlierInst); + + if (earliestUser && earliestUser->comesBefore(laterInst)) { + return nullptr; + } + + b.SetInsertPoint(laterInst); + + auto *scalarVec = b.CreateVectorSplat(2, PoisonValue::get(b.getHalfTy())); + scalarVec = b.CreateInsertElement(scalarVec, timesScalarLo->getArgOperand(1), b.getInt32(0)); + scalarVec = b.CreateInsertElement(scalarVec, timesScalarHi->getArgOperand(1), b.getInt32(1)); + auto *timesScalarPacked = + b.CreateCoopMatrixTimesScalar(packedMatrix, scalarVec, Builder::CooperativeMatrixElementType::Float16Packed, + Builder::CooperativeMatrixLayout::AccumulatorMatrixLayout); + m_eraseList.push_back(timesScalarLo); + m_eraseList.push_back(timesScalarHi); + return timesScalarPacked; +} + // ===================================================================================================================== // Run the pass on a function. // @@ -605,7 +932,10 @@ bool CooperativeMatrixCombiner::tryFoldComponentContaining(Value *start) { // @param [in/out] analysisManager : Analysis manager to use for this transformation // @returns : The preserved analyses (The Analyses that are still valid after this pass) PreservedAnalyses CombineCooperativeMatrix::run(Function &function, FunctionAnalysisManager &analysisManager) { - CooperativeMatrixCombiner combiner{function}; + const auto &moduleAnalysisManager = analysisManager.getResult(function); + PipelineState *pipelineState = + moduleAnalysisManager.getCachedResult(*function.getParent())->getPipelineState(); + CooperativeMatrixCombiner combiner{function, pipelineState->getTargetInfo().getGfxIpVersion()}; if (combiner.run()) { PreservedAnalyses PA; diff --git a/lgc/patch/FragColorExport.cpp b/lgc/patch/FragColorExport.cpp index 40ae489eab..2899035029 100644 --- a/lgc/patch/FragColorExport.cpp +++ b/lgc/patch/FragColorExport.cpp @@ -95,9 +95,10 @@ static void extractElements(Value *input, BuilderBase &builder, std::arraygetType(); @@ -127,9 +128,6 @@ Value *FragColorExport::handleColorExportInstructions(Value *output, unsigned hw Type *exportTy = exportTypeMapping[expFmt]; - const bool dualSourceBlendedEnable = m_pipelineState->getColorExportState().dualSourceBlendEnable || - m_pipelineState->getColorExportState().dynamicDualSourceBlendEnable; - // For 32bit output, we always to scalarize, but for 16bit output we may just operate on vector. if (exportTy->isFloatTy()) { if (compCount == 1) { @@ -243,7 +241,7 @@ Value *FragColorExport::handleColorExportInstructions(Value *output, unsigned hw } if (m_pipelineState->getTargetInfo().getGfxIpVersion().major >= 11) { - if (dualSourceBlendedEnable) { + if (isDualSource) { // Save them for later dual-source-swizzle m_blendSourceChannels = exportTy->isHalfTy() ? (compCount + 1) / 2 : compCount; assert(hwColorExport <= 1); @@ -466,9 +464,21 @@ bool LowerFragColorExport::runImpl(Module &module, PipelineShadersResult &pipeli collectExportInfoForBuiltinOutput(fragEntryPoint, builder); collectExportInfoForGenericOutputs(fragEntryPoint, builder); + // Now do color exports by color buffer. + // Read the dynamicDualSourceBlend value from user data and jump to different branch basing on the value + // 1. For dynamic pipeline, as the colorblendequation may be dynamic state, so there is a case like this: + // Create blendEquation(only use srcColor) -> bindDynamicPipeline -> setBlendEquation(use srcColor1). + // As a result, it needs switch the export instruction from normal export into dual exports. + // 2. For static pipeline which will not use the usedata and identify whether do dualSourceBlend + // Just according to the dualSourceBlendEnable flag. + Value *dynamicIsDualSource = builder.getInt32(0); + if (m_pipelineState->getTargetInfo().getGfxIpVersion().major >= 11) { + dynamicIsDualSource = ShaderInputs::getSpecialUserData(UserDataMapping::DynamicDualSrcBlendInfo, builder); + } + bool willGenerateColorExportShader = m_pipelineState->isUnlinked() && !m_pipelineState->hasColorExportFormats(); if (willGenerateColorExportShader && !m_info.empty()) { - createTailJump(fragEntryPoint, builder); + createTailJump(fragEntryPoint, builder, dynamicIsDualSource); return true; } @@ -476,7 +486,7 @@ bool LowerFragColorExport::runImpl(Module &module, PipelineShadersResult &pipeli bool dummyExport = (m_pipelineState->getTargetInfo().getGfxIpVersion().major < 10 || m_resUsage->builtInUsage.fs.discard); fragColorExport.generateExportInstructions(m_info, m_exportValues, dummyExport, m_pipelineState->getPalMetadata(), - builder); + builder, dynamicIsDualSource); return !m_info.empty() || dummyExport; } @@ -588,7 +598,7 @@ void LowerFragColorExport::collectExportInfoForGenericOutputs(Function *fragEntr // // @param fragEntryPoint : The fragment shader to which we should add the export instructions. // @param builder : The builder object that will be used to create new instructions. -void LowerFragColorExport::createTailJump(Function *fragEntryPoint, BuilderBase &builder) { +void LowerFragColorExport::createTailJump(Function *fragEntryPoint, BuilderBase &builder, Value *isDualSource) { // Add the export info to be used when linking shaders to generate the color export shader and compute the spi shader // color format in the metadata. m_pipelineState->getPalMetadata()->addColorExportInfo(m_info); @@ -599,10 +609,11 @@ void LowerFragColorExport::createTailJump(Function *fragEntryPoint, BuilderBase for (const ColorExportInfo &info : m_info) { outputTypes.push_back(getVgprTy(info.ty)); } - Type *retTy = StructType::get(*m_context, outputTypes); + outputTypes.push_back(builder.getInt32Ty()); // Now build the return value. - Value *retVal = PoisonValue::get(retTy); + SmallVector cesArgs; + unsigned returnLocation = 0; for (unsigned idx = 0; idx < m_info.size(); ++idx) { const ColorExportInfo &info = m_info[idx]; @@ -612,14 +623,14 @@ void LowerFragColorExport::createTailJump(Function *fragEntryPoint, BuilderBase continue; if (output->getType() != outputTypes[idx]) output = generateValueForOutput(output, outputTypes[idx], builder); - retVal = builder.CreateInsertValue(retVal, output, returnLocation); + cesArgs.push_back(output); ++returnLocation; } + cesArgs.push_back(isDualSource); if (m_pipelineState->getOptions().enableColorExportShader) { // Build color export function type - auto funcTy = FunctionType::get(builder.getVoidTy(), {retTy}, false); - + auto funcTy = FunctionType::get(builder.getVoidTy(), outputTypes, false); // Convert color export shader address to function pointer auto funcTyPtr = funcTy->getPointerTo(ADDR_SPACE_CONST); auto colorShaderAddr = ShaderInputs::getSpecialUserData(UserDataMapping::ColorExportAddr, builder); @@ -627,15 +638,20 @@ void LowerFragColorExport::createTailJump(Function *fragEntryPoint, BuilderBase auto funcPtr = addrExt.extendWithPc(colorShaderAddr, funcTyPtr, builder); // Jump - auto callInst = builder.CreateCall(funcTy, funcPtr, retVal); + auto callInst = builder.CreateCall(funcTy, funcPtr, cesArgs); callInst->setCallingConv(CallingConv::AMDGPU_Gfx); + callInst->addParamAttr(returnLocation, Attribute::InReg); callInst->setDoesNotReturn(); callInst->setOnlyWritesMemory(); } else { + Type *retTy = StructType::get(*m_context, outputTypes); addFunctionArgs(fragEntryPoint, retTy, {}, {}); - - auto *retInst = builder.GetInsertPoint()->getParent()->getTerminator(); + Value *retVal = PoisonValue::get(retTy); + for (int idx = 0; idx < cesArgs.size(); ++idx) { + retVal = builder.CreateInsertValue(retVal, cesArgs[idx], idx); + } retVal = builder.CreateRet(retVal); + auto *retInst = builder.GetInsertPoint()->getParent()->getTerminator(); retInst->eraseFromParent(); } } @@ -854,17 +870,61 @@ Value *FragColorExport::dualSourceSwizzle(BuilderBase &builder) { } // ===================================================================================================================== -// Generates the export instructions based on the given color export information. +// Update the color export information when enableFragColor is set. +// +// @param originExpinfo : The original color export information for each color export in no particular order. +// @param pCbShaderMask: The cbShaderMask after update color export information +// @param [out] outExpinfo : The updated color export information when enableFragColor is true. +void FragColorExport::updateColorExportInfoWithBroadCastInfo(ArrayRef originExpinfo, + SmallVector &outExpinfo, + unsigned *pCbShaderMask) { + // As enableFragColor will only be enabled by OGL, so it will not consider on the dualSource cases. + SmallVector broadCastInfo; + if (m_pipelineState->getOptions().enableFragColor) { + auto &expInfo = originExpinfo[0]; + assert(expInfo.ty != nullptr); + for (unsigned location = 0; location < MaxColorTargets; ++location) { + if (m_pipelineState->getColorExportFormat(location).dfmt != BufDataFormatInvalid) + broadCastInfo.push_back({0, location, expInfo.isSigned, expInfo.ty}); + } + } + outExpinfo = + m_pipelineState->getOptions().enableFragColor ? broadCastInfo : SmallVector(originExpinfo); + for (auto &exp : outExpinfo) { + if (exp.hwColorTarget == MaxColorTargets) + continue; + const unsigned channelWriteMask = m_pipelineState->getColorExportFormat(exp.location).channelWriteMask; + unsigned gfxIp = m_pipelineState->getTargetInfo().getGfxIpVersion().major; + bool needUpdateMask = false; + if (exp.location == 0 || gfxIp > 10) { + needUpdateMask = (m_pipelineState->computeExportFormat(exp.ty, exp.location) != 0) && + (channelWriteMask > 0 || m_pipelineState->getColorExportState().alphaToCoverageEnable); + } else { + needUpdateMask = (m_pipelineState->computeExportFormat(exp.ty, exp.location) != 0) && (channelWriteMask > 0); + } + if (needUpdateMask) { + // For dualSource, the cbShaderMask will only be valid for location=0, other locations setting will be + // redundant. ToDo: this point can be optimized when use different ShaderMaskMetaKey or compile different + // shaders. + *pCbShaderMask |= (0xF << (4 * exp.location)); + } + } +} + +// ===================================================================================================================== +// Generates the export instructions based on the given color export information in dynamic state // // @param info : The color export information for each color export in no particular order. // @param values : The values that are to be exported. Indexed by the hw color target. // @param exportFormat : The export format for each color target. Indexed by the hw color target. // @param [out] palMetadata : The PAL metadata that will be extended with relevant information. // @param builder : The builder object that will be used to create new instructions. +// @param dynamicIsDualSource: Identify whether it's in dynamicDualSourceBlend state void FragColorExport::generateExportInstructions(ArrayRef info, ArrayRef values, - bool dummyExport, PalMetadata *palMetadata, BuilderBase &builder) { - SmallVector finalExportFormats; + bool dummyExport, PalMetadata *palMetadata, BuilderBase &builder, + Value *dynamicIsDualSource) { Value *lastExport = nullptr; + unsigned gfxip = m_pipelineState->getTargetInfo().getGfxIpVersion().major; // MRTZ export comes first if it exists (this is a HW requirement on gfx11+ and an optional good idea on earlier HW). // We make the assume here that it is also first in the info list. @@ -873,8 +933,7 @@ void FragColorExport::generateExportInstructions(ArrayRef info, // Depth export alpha comes from MRT0.a if there is MRT0.a and A2C is enabled on GFX11+ Value *alpha = PoisonValue::get(Type::getFloatTy(*m_context)); - if (!dummyExport && m_pipelineState->getTargetInfo().getGfxIpVersion().major >= 11 && - m_pipelineState->getColorExportState().alphaToCoverageEnable) { + if (!dummyExport && gfxip >= 11 && m_pipelineState->getColorExportState().alphaToCoverageEnable) { for (auto &curInfo : info) { if (curInfo.location != 0) continue; @@ -920,57 +979,106 @@ void FragColorExport::generateExportInstructions(ArrayRef info, info = info.drop_front(1); } - // Record each color buffer's export info for broadcasting - llvm::SmallVector broadCastInfo; - if (m_pipelineState->getOptions().enableFragColor) { - auto &expInfo = info[0]; - assert(expInfo.ty != nullptr); + SmallVector finalExportFormats; + SmallVector finalExpInfo; + unsigned cbShaderMask = 0; + + ReturnInst *retInst = cast(builder.GetInsertPoint()->getParent()->getTerminator()); + Function *originFunc = builder.GetInsertPoint()->getParent()->getParent(); + BasicBlock *dualSourceBlock = nullptr; + BasicBlock *normalExportBlock = nullptr; + + updateColorExportInfoWithBroadCastInfo(info, finalExpInfo, &cbShaderMask); + + if (m_pipelineState->getColorExportState().dualSourceBlendDynamicEnable && (gfxip >= 11)) { + // For dynamiceState, whether do dualSourceBlend will depend on the user data. + dualSourceBlock = BasicBlock::Create(*m_context, "dualSourceSwizzle", originFunc); + normalExportBlock = BasicBlock::Create(*m_context, "normalExport", originFunc); + Value *staticDualEnable = builder.getInt32(m_pipelineState->getColorExportState().dualSourceBlendEnable); + Value *isDualSource = builder.CreateOr(dynamicIsDualSource, + builder.CreateAnd(staticDualEnable, builder.CreateNot(dynamicIsDualSource))); + isDualSource = builder.CreateICmpNE(dynamicIsDualSource, builder.getInt32(0)); + builder.CreateCondBr(isDualSource, dualSourceBlock, normalExportBlock); + } else { + if (retInst) { + retInst->eraseFromParent(); + retInst = nullptr; + } + if (m_pipelineState->getColorExportState().dualSourceBlendEnable && (gfxip >= 11)) + // For only-static case, it will depend on dualSourceBlendEnable flag to do dualSourceBlend. + dualSourceBlock = builder.GetInsertBlock(); + else + // Both staticDualSourceBlend flag and dynamicDualSourceBlend flag are 0, don't export in dualSourceBlend. + normalExportBlock = builder.GetInsertBlock(); + } + // Construct ".dualSourceSwizzle" Block, only construct when the dynamicEnable is on and staticValue is true. + if ((m_pipelineState->getColorExportState().dualSourceBlendDynamicEnable || + m_pipelineState->getColorExportState().dualSourceBlendEnable) && + (gfxip >= 11)) { + builder.SetInsertPoint(dualSourceBlock); + unsigned hwColorExport = 0; for (unsigned location = 0; location < MaxColorTargets; ++location) { - if (m_pipelineState->getColorExportFormat(location).dfmt != BufDataFormatInvalid) - broadCastInfo.push_back({0, location, expInfo.isSigned, expInfo.ty}); + auto infoIt = llvm::find_if( + finalExpInfo, [&](const ColorExportInfo &finalExpInfo) { return finalExpInfo.location == location; }); + if (infoIt == finalExpInfo.end()) + continue; + assert(infoIt->hwColorTarget < MaxColorTargets); + auto expFmt = static_cast(m_pipelineState->computeExportFormat(infoIt->ty, location, true)); + const unsigned channelWriteMask = m_pipelineState->getColorExportFormat(location, true).channelWriteMask; + bool needExpInst = (expFmt != EXP_FORMAT_ZERO) && + (channelWriteMask > 0 || m_pipelineState->getColorExportState().alphaToCoverageEnable); + if (needExpInst) { + // Collect info for dualSourceBlend and save then in m_blendSources, so set the last parameter=true; + handleColorExportInstructions(values[infoIt->hwColorTarget], hwColorExport, builder, expFmt, infoIt->isSigned, + channelWriteMask, true); + finalExportFormats.push_back(expFmt); + ++hwColorExport; + } } + lastExport = dualSourceSwizzle(builder); + FragColorExport::setDoneFlag(lastExport, builder); + builder.CreateRetVoid(); } - // Now do color exports by color buffer. - unsigned hwColorExport = 0; - auto finalExpInfo = m_pipelineState->getOptions().enableFragColor ? ArrayRef(broadCastInfo) : info; - for (unsigned location = 0; location < MaxColorTargets; ++location) { - auto infoIt = llvm::find_if(finalExpInfo, - [&](const ColorExportInfo &finalExpInfo) { return finalExpInfo.location == location; }); - if (infoIt == finalExpInfo.end()) - continue; - - assert(infoIt->hwColorTarget < MaxColorTargets); - auto expFmt = static_cast(m_pipelineState->computeExportFormat(infoIt->ty, location)); - unsigned channelWriteMask = m_pipelineState->getColorExportFormat(location).channelWriteMask; - bool needExpInst = (expFmt != EXP_FORMAT_ZERO) && - (channelWriteMask > 0 || m_pipelineState->getColorExportState().alphaToCoverageEnable); - if (needExpInst) { - lastExport = handleColorExportInstructions(values[infoIt->hwColorTarget], hwColorExport, builder, expFmt, - infoIt->isSigned, channelWriteMask); - finalExportFormats.push_back(expFmt); - ++hwColorExport; + // Construct ".normalExport" Block + if (m_pipelineState->getColorExportState().dualSourceBlendDynamicEnable || + !m_pipelineState->getColorExportState().dualSourceBlendEnable || (gfxip < 11)) { + builder.SetInsertPoint(normalExportBlock); + unsigned hwColorExport = 0; + for (unsigned location = 0; location < MaxColorTargets; ++location) { + auto infoIt = llvm::find_if( + finalExpInfo, [&](const ColorExportInfo &finalExpInfo) { return finalExpInfo.location == location; }); + if (infoIt == finalExpInfo.end()) + continue; + assert(infoIt->hwColorTarget < MaxColorTargets); + const unsigned channelWriteMask = m_pipelineState->getColorExportFormat(location).channelWriteMask; + auto expFmt = static_cast(m_pipelineState->computeExportFormat(infoIt->ty, location)); + bool needExpInst = (expFmt != EXP_FORMAT_ZERO) && + (channelWriteMask > 0 || m_pipelineState->getColorExportState().alphaToCoverageEnable); + if (needExpInst) { + // Don't collect info for dualSourceBlend just do normal color export, so set the last parameter=false; + lastExport = handleColorExportInstructions(values[infoIt->hwColorTarget], hwColorExport, builder, expFmt, + infoIt->isSigned, channelWriteMask, false); + finalExportFormats.push_back(expFmt); + ++hwColorExport; + } } + if (!lastExport && dummyExport) { + lastExport = FragColorExport::addDummyExport(builder); + finalExportFormats.push_back(EXP_FORMAT_32_R); + } + if (lastExport) + FragColorExport::setDoneFlag(lastExport, builder); + builder.CreateRetVoid(); } - // Special case of color exports: dual swizzle on gfx11+. They are implicitly captured above, and this case implies - // no other color exports. (TODO: Cleanup the implicit capture; we should just check this up-front.) - if (m_pipelineState->getTargetInfo().getGfxIpVersion().major >= 11 && - (m_pipelineState->getColorExportState().dualSourceBlendEnable || - m_pipelineState->getColorExportState().dynamicDualSourceBlendEnable)) - lastExport = dualSourceSwizzle(builder); - - if (!lastExport && dummyExport) { - lastExport = FragColorExport::addDummyExport(builder); - finalExportFormats.push_back(EXP_FORMAT_32_R); + if (retInst) { + retInst->eraseFromParent(); } - if (lastExport) - FragColorExport::setDoneFlag(lastExport, builder); - palMetadata->updateSpiShaderColFormat(finalExportFormats); - palMetadata->updateCbShaderMask(info); + palMetadata->updateCbShaderMask(cbShaderMask); } // ===================================================================================================================== diff --git a/lgc/patch/LowerCooperativeMatrix.cpp b/lgc/patch/LowerCooperativeMatrix.cpp index 4582205f70..599fd0f04a 100644 --- a/lgc/patch/LowerCooperativeMatrix.cpp +++ b/lgc/patch/LowerCooperativeMatrix.cpp @@ -245,13 +245,24 @@ void LowerCooperativeMatrix::visitCallInst(CallInst &callInst) { Value *matrixC = callInst.getOperand(2); bool isSignedA = cast(callInst.getOperand(3))->getZExtValue(); bool isSignedB = cast(callInst.getOperand(4))->getZExtValue(); - bool isSat = cast(callInst.getOperand(5))->getZExtValue(); + bool isSatOrOpsel = cast(callInst.getOperand(5))->getZExtValue(); + bool isTied = cast(callInst.getOperand(6))->getZExtValue(); Builder::CooperativeMatrixElementType accumElemType = - static_cast(cast(callInst.getOperand(6))->getZExtValue()); - Builder::CooperativeMatrixElementType factorElemType = static_cast(cast(callInst.getOperand(7))->getZExtValue()); - Value *resultVal = cooperativeMatrixMulAdd(matrixA, matrixB, matrixC, isSignedA, isSignedB, isSat, accumElemType, - factorElemType, callInst.getName(), &callInst); + Builder::CooperativeMatrixElementType factorElemType = + static_cast(cast(callInst.getOperand(8))->getZExtValue()); + Value *resultVal = cooperativeMatrixMulAdd(matrixA, matrixB, matrixC, isSignedA, isSignedB, isSatOrOpsel, isTied, + accumElemType, factorElemType, callInst.getName(), &callInst); + callInst.replaceAllUsesWith(resultVal); + } else if (mangledName.startswith(lgcName::CooperativeMatrixPack)) { + Value *matrixA = callInst.getOperand(0); + Value *matrixB = callInst.getOperand(1); + Value *resultVal = cooperativeMatrixPack(matrixA, matrixB, callInst.getName(), &callInst); + callInst.replaceAllUsesWith(resultVal); + } else if (mangledName.startswith(lgcName::CooperativeMatrixUnpack)) { + Value *packedMatrix = callInst.getOperand(0); + bool high = cast(callInst.getOperand(1))->getZExtValue(); + Value *resultVal = cooperativeMatrixUnpack(packedMatrix, high, callInst.getName(), &callInst); callInst.replaceAllUsesWith(resultVal); } else { @@ -299,6 +310,7 @@ LowerCooperativeMatrix::getTypeProperties(Builder::CooperativeMatrixElementType props.numMatrixWords = 8; break; case Builder::CooperativeMatrixElementType::Float16: + case Builder::CooperativeMatrixElementType::Float16Packed: case Builder::CooperativeMatrixElementType::Int16: props.numMatrixElements = 16; props.numMatrixWords = 8; @@ -317,11 +329,15 @@ LowerCooperativeMatrix::getTypeProperties(Builder::CooperativeMatrixElementType elemType != Builder::CooperativeMatrixElementType::Int32); props.numFlatElements = 16; } else if (layout == Builder::CooperativeMatrixLayout::AccumulatorMatrixLayout) { - props.numFlatElements = waveSize == 32 ? 8 : 4; if (elemType == Builder::CooperativeMatrixElementType::Float16 || elemType == Builder::CooperativeMatrixElementType::Int16) { props.matrixElementStride = 2; } + if (elemType == Builder::CooperativeMatrixElementType::Float16Packed) { + props.numFlatElements = waveSize == 32 ? 16 : 8; + } else { + props.numFlatElements = waveSize == 32 ? 8 : 4; + } } else if (layout == Builder::CooperativeMatrixLayout::Gfx10AccumulatorMatrixLayout || layout == Builder::CooperativeMatrixLayout::Gfx10Accumulator16bitMatrixLayout) { props.numFlatElements = 8; @@ -776,7 +792,8 @@ Value *LowerCooperativeMatrix::cooperativeMatrixBinaryOp(Builder::CooperativeMat // Create cooperative matrix MatrixTimesScalar operation // // @param matrix : The first operand and it should be a cooperative matrix. -// @param scalar : The second operand and it should be a scalar. +// @param scalar : The second operand and it should be a scalar. If the matrix +// is a packed accumulator matrix, the scalar has to be a <2 x half> vector. // @param elemType : The component type of the matrix. // @param layout : Identify whether it's A/B or C/D // @param instName : Name to give instruction(s). @@ -791,10 +808,15 @@ Value *LowerCooperativeMatrix::coopMatrixTimesScalar(Value *matrix, Value *scala Value *vcFlat = convCoopMatrixVecToFlatVec(builder, matrix, elemType, layout); const unsigned numElems = cast(vcFlat->getType())->getNumElements(); - auto splat = builder.CreateVectorSplat(numElems, scalar); + const bool packedScalarVec = scalar->getType()->isVectorTy(); + const auto shuffleIndices = numElems == 16 ? SmallVector({0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1}) + : SmallVector({0, 1, 0, 1, 0, 1, 0, 1}); + auto splat = packedScalarVec ? builder.CreateShuffleVector(scalar, shuffleIndices) + : builder.CreateVectorSplat(numElems, scalar); Value *vcFlatResult; if ((elemType == Builder::CooperativeMatrixElementType::Float16) || - (elemType == Builder::CooperativeMatrixElementType::Float32)) { + (elemType == Builder::CooperativeMatrixElementType::Float32) || + (elemType == Builder::CooperativeMatrixElementType::Float16Packed)) { vcFlatResult = builder.CreateFMul(vcFlat, splat); } else { vcFlatResult = builder.CreateMul(vcFlat, splat); @@ -1490,7 +1512,7 @@ Value *LowerCooperativeMatrix::transposeCooperativeMatrixRecursively(llvm::Value // @param instName : Name to give instruction(s). // @param insertPos : Where to insert the instruction Value *LowerCooperativeMatrix::cooperativeMatrixMulAdd(llvm::Value *matrixA, llvm::Value *matrixB, llvm::Value *matrixC, - bool isSignedA, bool isSignedB, bool isSat, + bool isSignedA, bool isSignedB, bool isSatOrOpsel, bool isTied, Builder::CooperativeMatrixElementType accumElemType, Builder::CooperativeMatrixElementType factorElemType, const Twine &instName, Instruction *insertPos) { @@ -1555,17 +1577,24 @@ Value *LowerCooperativeMatrix::cooperativeMatrixMulAdd(llvm::Value *matrixA, llv } else if (factorElemType == Builder::CooperativeMatrixElementType::Int8 && accumElemType == Builder::CooperativeMatrixElementType::Int32) { - matrixD = builder.CreateIntrinsic( - matrixC->getType(), Intrinsic::amdgcn_wmma_i32_16x16x16_iu8, - {builder.getInt1(isSignedA), matrixA, builder.getInt1(isSignedB), matrixB, matrixC, builder.getInt1(isSat)}, - nullptr, instName); + matrixD = builder.CreateIntrinsic(matrixC->getType(), Intrinsic::amdgcn_wmma_i32_16x16x16_iu8, + {builder.getInt1(isSignedA), matrixA, builder.getInt1(isSignedB), matrixB, + matrixC, builder.getInt1(isSatOrOpsel)}, + nullptr, instName); } else if (factorElemType == Builder::CooperativeMatrixElementType::Float16 && accumElemType == Builder::CooperativeMatrixElementType::Float16) { // Matrix convert to match intrinsic arguments: Wave32: float32*v8->half*v16 // Wave64: float32*v4->half*v8 - matrixD = builder.CreateIntrinsic(matrixC->getType(), Intrinsic::amdgcn_wmma_f16_16x16x16_f16, - {matrixA, matrixB, matrixC, builder.getInt1(isSat)}, nullptr, instName); + auto intrinsic = Intrinsic::amdgcn_wmma_f16_16x16x16_f16; + if (isTied) +#if defined(LLVM_MAIN_REVISION) && LLVM_MAIN_REVISION < 479080 + llvm_unreachable("Tied intrinsics not implemented"); +#else + intrinsic = Intrinsic::amdgcn_wmma_f16_16x16x16_f16_tied; +#endif + matrixD = builder.CreateIntrinsic(matrixC->getType(), intrinsic, + {matrixA, matrixB, matrixC, builder.getInt1(isSatOrOpsel)}, nullptr, instName); } else { llvm_unreachable("The accumulator type is not supported."); } @@ -1618,11 +1647,13 @@ Value *LowerCooperativeMatrix::cooperativeMatrixMulAdd(llvm::Value *matrixA, llv Value *mulAB; Value *initAccumulator = builder.CreateExtractElement(matrixC, idxc); if (factorElemType == Builder::CooperativeMatrixElementType::Float16) { - mulAB = createDotProductFp16Fp32(rowData, matrixB, initAccumulator, isSat, instName, insertPos); + mulAB = createDotProductFp16Fp32(rowData, matrixB, initAccumulator, isSatOrOpsel, instName, insertPos); } else if (factorElemType == Builder::CooperativeMatrixElementType::Int16) { - mulAB = createDotProductInt16Int32(rowData, matrixB, initAccumulator, flags, isSat, instName, insertPos); + mulAB = + createDotProductInt16Int32(rowData, matrixB, initAccumulator, flags, isSatOrOpsel, instName, insertPos); } else if (factorElemType == Builder::CooperativeMatrixElementType::Int8) { - mulAB = createDotProductInt8Int32(rowData, matrixB, initAccumulator, flags, isSat, instName, insertPos); + mulAB = + createDotProductInt8Int32(rowData, matrixB, initAccumulator, flags, isSatOrOpsel, instName, insertPos); } else { llvm_unreachable("Unsupported element type!"); } @@ -1658,11 +1689,13 @@ Value *LowerCooperativeMatrix::cooperativeMatrixMulAdd(llvm::Value *matrixA, llv Value *accumulator2 = builder.CreateExtractElement(matrixC, accIdx + 1); if (accumElemType == Builder::CooperativeMatrixElementType::Float16) { - mulAB1 = createDotProductFp16Fp16(rowData1, colData, accumulator1, isSat, instName, insertPos); - mulAB2 = createDotProductFp16Fp16(rowData2, colData, accumulator2, isSat, instName, insertPos); + mulAB1 = createDotProductFp16Fp16(rowData1, colData, accumulator1, isSatOrOpsel, instName, insertPos); + mulAB2 = createDotProductFp16Fp16(rowData2, colData, accumulator2, isSatOrOpsel, instName, insertPos); } else { - mulAB1 = createDotProductInt16Int16(rowData1, colData, accumulator1, flags, isSat, instName, insertPos); - mulAB2 = createDotProductInt16Int16(rowData2, colData, accumulator2, flags, isSat, instName, insertPos); + mulAB1 = + createDotProductInt16Int16(rowData1, colData, accumulator1, flags, isSatOrOpsel, instName, insertPos); + mulAB2 = + createDotProductInt16Int16(rowData2, colData, accumulator2, flags, isSatOrOpsel, instName, insertPos); } dotProductValue = builder.CreateInsertElement(dotProductValue, mulAB1, accIdx); dotProductValue = builder.CreateInsertElement(dotProductValue, mulAB2, accIdx + 1); @@ -1884,6 +1917,52 @@ Value *LowerCooperativeMatrix::createDotProductInt16Int16(Value *vector1, Value return scalar; } +// ===================================================================================================================== +// Create code to pack two accumulator matrices into one set of registers +// +// @param matrixCLo : The lower accumulator +// @param matrixCHi : The higher accumulator +// @param instName : Name to give instruction(s) +// @param insertPos : Where to insert the instruction +Value *LowerCooperativeMatrix::cooperativeMatrixPack(llvm::Value *matrixCLo, llvm::Value *matrixCHi, + const Twine &instName, Instruction *insertPos) { + BuilderBase builder(*m_context); + builder.SetInsertPoint(insertPos); + + static const int shuffleIndices[] = {0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30}; + + const auto halfVecTy = FixedVectorType::get(builder.getHalfTy(), 16); + matrixCLo = builder.CreateBitCast(matrixCLo, halfVecTy); + matrixCHi = builder.CreateBitCast(matrixCHi, halfVecTy); + + auto result = builder.CreateShuffleVector(matrixCLo, matrixCHi, shuffleIndices); + + return builder.CreateBitCast(result, FixedVectorType::get(builder.getFloatTy(), 8)); +} + +// ===================================================================================================================== +// Create code to unpack one packed accumulator matrix into two separate set of +// registers +// +// @param packedMatrix : The packed accumulator matrix +// @param: high: Whether to get the matrix in the upper half of the registers +// @param instName : Name to give instruction(s) +// @param insertPos : Where to insert the instruction +Value *LowerCooperativeMatrix::cooperativeMatrixUnpack(llvm::Value *packedMatrix, bool high, const Twine &instName, + Instruction *insertPos) { + BuilderBase builder(*m_context); + builder.SetInsertPoint(insertPos); + + static const int shuffleIndicesLo[] = {0, -1, 2, -1, 4, -1, 6, -1, 8, -1, 10, -1, 12, -1, 14, -1}; + static const int shuffleIndicesHi[] = {1, -1, 3, -1, 5, -1, 7, -1, 9, -1, 11, -1, 13, -1, 15, -1}; + + const auto halfVecTy = FixedVectorType::get(builder.getHalfTy(), 16); + auto matrixPackedCast = builder.CreateBitCast(packedMatrix, halfVecTy); + auto matrixUnpacked = builder.CreateShuffleVector(matrixPackedCast, high ? shuffleIndicesHi : shuffleIndicesLo); + + return builder.CreateBitCast(matrixUnpacked, FixedVectorType::get(builder.getFloatTy(), 8)); +} + // ===================================================================================================================== // Get lane id. // @param builder : The IR builder to create and insert IR instruction diff --git a/lgc/patch/NggPrimShader.cpp b/lgc/patch/NggPrimShader.cpp index dd03afe840..10c97ca880 100644 --- a/lgc/patch/NggPrimShader.cpp +++ b/lgc/patch/NggPrimShader.cpp @@ -7482,7 +7482,7 @@ Value *NggPrimShader::fetchXfbOutput(Function *target, ArrayRef args // If we don't clone the target function, we are going to run it and handle vertex attribute through memory here. if (dontClone) { - // Setup attribute ring base and relative vertx index in subgroup as two additional arguments to export vertex + // Setup attribute ring base and relative vertex index in subgroup as two additional arguments to export vertex // attributes through memory if (m_gfxIp.major >= 11 && !m_hasGs) { // For GS, vertex attribute exports are in copy shader const auto attribCount = diff --git a/lgc/patch/NggPrimShader.h b/lgc/patch/NggPrimShader.h index a69aa1ef12..3c9defd434 100644 --- a/lgc/patch/NggPrimShader.h +++ b/lgc/patch/NggPrimShader.h @@ -224,7 +224,7 @@ class NggPrimShader { void loadStreamOutBufferInfo(llvm::Value *userData); void distributePrimitiveId(llvm::Value *primitiveId); - llvm::Value *cullPrimitive(llvm::Value *vertxIndex0, llvm::Value *vertxIndex1, llvm::Value *vertxIndex2); + llvm::Value *cullPrimitive(llvm::Value *vertexIndex0, llvm::Value *vertexIndex1, llvm::Value *vertexIndex2); void sendGsAllocReqMessage(); void exportPassthroughPrimitive(); void exportPrimitive(llvm::Value *primitiveCulled); @@ -289,9 +289,9 @@ class NggPrimShader { llvm::Value *ballot(llvm::Value *value); - llvm::Value *fetchVertexPositionData(llvm::Value *vertxIndex); - llvm::Value *fetchCullDistanceSignMask(llvm::Value *vertxIndex); - llvm::Value *calcVertexItemOffset(unsigned streamId, llvm::Value *vertxIndex); + llvm::Value *fetchVertexPositionData(llvm::Value *vertexIndex); + llvm::Value *fetchCullDistanceSignMask(llvm::Value *vertexIndex); + llvm::Value *calcVertexItemOffset(unsigned streamId, llvm::Value *vertexIndex); void processVertexAttribExport(llvm::Function *&target); @@ -300,8 +300,8 @@ class NggPrimShader { llvm::Value *fetchXfbOutput(llvm::Function *target, llvm::ArrayRef args, llvm::SmallVector &xfbOutputExports); - llvm::Value *readXfbOutputFromLds(llvm::Type *readDataTy, llvm::Value *vertxIndex, unsigned outputIndex); - void writeXfbOutputToLds(llvm::Value *writeData, llvm::Value *vertxIndex, unsigned outputIndex); + llvm::Value *readXfbOutputFromLds(llvm::Type *readDataTy, llvm::Value *vertexIndex, unsigned outputIndex); + void writeXfbOutputToLds(llvm::Value *writeData, llvm::Value *vertexIndex, unsigned outputIndex); // Checks if NGG culling operations are enabled bool enableCulling() const { diff --git a/lgc/patch/PatchEntryPointMutate.cpp b/lgc/patch/PatchEntryPointMutate.cpp index 70a594750f..977f3c4e90 100644 --- a/lgc/patch/PatchEntryPointMutate.cpp +++ b/lgc/patch/PatchEntryPointMutate.cpp @@ -1793,6 +1793,12 @@ void PatchEntryPointMutate::addSpecialUserDataArgs(SmallVectorImpl specialUserDataArgs.push_back(UserDataArg(builder.getInt32Ty(), "sampleInfo", UserDataMapping::SampleInfo, &intfData->entryArgIdxs.fs.sampleInfo)); } + + if (userDataUsage->isSpecialUserDataUsed(UserDataMapping::DynamicDualSrcBlendInfo)) { + specialUserDataArgs.push_back(UserDataArg(builder.getInt32Ty(), "dualSourceBlendUpdateInfo", + UserDataMapping::DynamicDualSrcBlendInfo, + &intfData->entryArgIdxs.fs.dynamicDualSrcBlendInfo)); + } } // Allocate register for stream-out buffer table, to go before the user data node args (unlike all the ones diff --git a/lgc/patch/PatchInOutImportExport.cpp b/lgc/patch/PatchInOutImportExport.cpp index 89b011e6f8..e105ab47de 100644 --- a/lgc/patch/PatchInOutImportExport.cpp +++ b/lgc/patch/PatchInOutImportExport.cpp @@ -35,6 +35,7 @@ #include "lgc/BuiltIns.h" #include "lgc/LgcDialect.h" #include "lgc/state/AbiUnlinked.h" +#include "lgc/state/PalMetadata.h" #include "lgc/state/PipelineShaders.h" #include "lgc/util/Debug.h" #include "llvm/IR/InlineAsm.h" @@ -482,12 +483,14 @@ void PatchInOutImportExport::processShader() { while (!func.use_empty()) { CallInst *reconfigCall = cast(*func.user_begin()); Value *localInvocationId = reconfigCall->getArgOperand(0); - bool isHwLocalInvocationId = dyn_cast(reconfigCall->getArgOperand(1))->getZExtValue(); - if ((layout.microLayout == WorkgroupLayout::Quads) || - (layout.macroLayout == WorkgroupLayout::SexagintiQuads)) { - localInvocationId = - reconfigWorkgroupLayout(localInvocationId, layout.macroLayout, layout.microLayout, workgroupSizeX, - workgroupSizeY, workgroupSizeZ, isHwLocalInvocationId, reconfigCall); + if (m_gfxIp.major <= 11) { + bool isHwLocalInvocationId = cast(reconfigCall->getArgOperand(1))->getZExtValue(); + if ((layout.microLayout == WorkgroupLayout::Quads) || + (layout.macroLayout == WorkgroupLayout::SexagintiQuads)) { + localInvocationId = + reconfigWorkgroupLayout(localInvocationId, layout.macroLayout, layout.microLayout, workgroupSizeX, + workgroupSizeY, workgroupSizeZ, isHwLocalInvocationId, reconfigCall); + } } reconfigCall->replaceAllUsesWith(localInvocationId); reconfigCall->eraseFromParent(); diff --git a/lgc/patch/ShaderInputs.cpp b/lgc/patch/ShaderInputs.cpp index 4b411a3ce4..3f739340c2 100644 --- a/lgc/patch/ShaderInputs.cpp +++ b/lgc/patch/ShaderInputs.cpp @@ -75,6 +75,8 @@ const char *ShaderInputs::getSpecialUserDataName(UserDataMapping kind) { return "StreamOutControlBuf"; case UserDataMapping::ColorExportAddr: return "ColorExportAddr"; + case UserDataMapping::DynamicDualSrcBlendInfo: + return "DualSourceBlendUpdateInfo"; default: return ""; } diff --git a/lgc/patch/ShaderMerger.cpp b/lgc/patch/ShaderMerger.cpp index 6313848ae4..bcf64cfa8a 100644 --- a/lgc/patch/ShaderMerger.cpp +++ b/lgc/patch/ShaderMerger.cpp @@ -1356,7 +1356,7 @@ void ShaderMerger::storeTessFactorsWithOpt(Value *threadIdInWave, IRBuilder<> &b auto userData = getFunctionArgument(entryPoint, NumSpecialSgprInputs); auto globalTable = builder.CreateExtractElement( - userData, static_cast(0)); // The first element of user data argument is always internal global tabl + userData, static_cast(0)); // The first element of user data argument is always internal global table Value *pc = builder.CreateIntrinsic(Intrinsic::amdgcn_s_getpc, {}, {}); pc = builder.CreateBitCast(pc, FixedVectorType::get(builder.getInt32Ty(), 2)); diff --git a/lgc/state/PalMetadata.cpp b/lgc/state/PalMetadata.cpp index edcf8624d0..b6ce24ad12 100644 --- a/lgc/state/PalMetadata.cpp +++ b/lgc/state/PalMetadata.cpp @@ -1044,18 +1044,7 @@ void PalMetadata::updateSpiShaderColFormat(ArrayRef expFormats) { // ===================================================================================================================== // Updates the CB_SHADER_MASK entry. // -void PalMetadata::updateCbShaderMask(llvm::ArrayRef exps) { - unsigned cbShaderMask = 0; - for (auto &exp : exps) { - if (exp.hwColorTarget == MaxColorTargets) - continue; - unsigned channelWriteMask = m_pipelineState->getColorExportFormat(exp.location).channelWriteMask; - bool needUpdateMask = (m_pipelineState->computeExportFormat(exp.ty, exp.location) != 0) && - (channelWriteMask > 0 || m_pipelineState->getColorExportState().alphaToCoverageEnable); - if (needUpdateMask) - cbShaderMask |= (0xF << (4 * exp.location)); - } - +void PalMetadata::updateCbShaderMask(unsigned cbShaderMask) { if (m_pipelineState->useRegisterFieldFormat()) { auto cbShaderMaskNode = m_pipelineNode[Util::Abi::PipelineMetadataKey::GraphicsRegisters] .getMap(true)[Util::Abi::GraphicsRegisterMetadataKey::CbShaderMask] diff --git a/lgc/state/PipelineState.cpp b/lgc/state/PipelineState.cpp index 34efe6768d..a1d9b57a0c 100644 --- a/lgc/state/PipelineState.cpp +++ b/lgc/state/PipelineState.cpp @@ -1199,8 +1199,9 @@ void PipelineState::setColorExportState(ArrayRef formats, con // Get format for one color export // // @param location : Export location -const ColorExportFormat &PipelineState::getColorExportFormat(unsigned location) { - if (getColorExportState().dualSourceBlendEnable || getColorExportState().dynamicDualSourceBlendEnable) +// @param dsBlendUpdate: The new value for the dynamicDualSourceBlend +const ColorExportFormat &PipelineState::getColorExportFormat(unsigned location, bool isDynamicDualSourceBlend) { + if (getColorExportState().dualSourceBlendEnable || isDynamicDualSourceBlend) location = 0; if (location >= m_colorExportFormats.size()) { @@ -1616,8 +1617,9 @@ InterfaceData *PipelineState::getShaderInterfaceData(ShaderStage shaderStage) { // // @param outputTy : Color output type // @param location : Location -unsigned PipelineState::computeExportFormat(Type *outputTy, unsigned location) { - const ColorExportFormat *colorExportFormat = &getColorExportFormat(location); +// @param isDynamicDualSrcBlend: Identify whether do dualSourceBlend +unsigned PipelineState::computeExportFormat(Type *outputTy, unsigned location, bool isDynamicDualSrcBlend) { + const ColorExportFormat *colorExportFormat = &getColorExportFormat(location, isDynamicDualSrcBlend); unsigned outputMask = outputTy->isVectorTy() ? (1 << cast(outputTy)->getNumElements()) - 1 : 1; const auto cbState = &getColorExportState(); // NOTE: Alpha-to-coverage only takes effect for outputs from color target 0. @@ -1625,8 +1627,7 @@ unsigned PipelineState::computeExportFormat(Type *outputTy, unsigned location) { // format. const bool enableAlphaToCoverage = (cbState->alphaToCoverageEnable && - ((location == 0) || - ((location == 1) && (cbState->dualSourceBlendEnable || cbState->dynamicDualSourceBlendEnable)))); + ((location == 0) || ((location == 1) && (cbState->dualSourceBlendEnable || isDynamicDualSrcBlend)))); const bool blendEnabled = colorExportFormat->blendEnable; diff --git a/lgc/test/CMakeLists.txt b/lgc/test/CMakeLists.txt index 88eed715fd..ddc9ea08e4 100644 --- a/lgc/test/CMakeLists.txt +++ b/lgc/test/CMakeLists.txt @@ -32,7 +32,16 @@ set(LGC_TEST_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) # required by configure_lit_site_cfg set(LLVM_LIT_OUTPUT_DIR ${LLVM_TOOLS_BINARY_DIR}) -get_target_property(LIT_DEFINITIONS LLVMlgc INTERFACE_COMPILE_DEFINITIONS) + +# When cross-compiling (or compiling a release tablegen), external llvm projects are defined but other projects are not. +# We expect that tests are not run in a cmake configuration where llpc_version does not exist, so we only use its +# definitions when it is available. +if (TARGET llpc_version) + get_target_property(LGC_LIT_DEFINITIONS LLVMlgc COMPILE_DEFINITIONS) + get_target_property(LIT_DEFINITIONS llpc_version INTERFACE_COMPILE_DEFINITIONS) + list(APPEND LIT_DEFINITIONS ${LGC_LIT_DEFINITIONS}) +endif() + configure_lit_site_cfg( ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py diff --git a/lgc/test/Transforms/CombineCooperativeMatrix/constants.lgc b/lgc/test/Transforms/CombineCooperativeMatrix/constants.lgc index 6fcd242e9e..9c5b754480 100644 --- a/lgc/test/Transforms/CombineCooperativeMatrix/constants.lgc +++ b/lgc/test/Transforms/CombineCooperativeMatrix/constants.lgc @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool lgc -; RUN: lgc -o - -passes=lgc-combine-cooperative-matrix %s | FileCheck --check-prefixes=CHECK %s +; RUN: lgc -o - -passes='require,function(lgc-combine-cooperative-matrix)' %s | FileCheck --check-prefixes=CHECK %s define <8 x float> @transpose_undef() { ; CHECK-LABEL: @transpose_undef( diff --git a/lgc/test/Transforms/CombineCooperativeMatrix/matmul-loop.lgc b/lgc/test/Transforms/CombineCooperativeMatrix/matmul-loop.lgc index 95e479a498..27adadc0b1 100644 --- a/lgc/test/Transforms/CombineCooperativeMatrix/matmul-loop.lgc +++ b/lgc/test/Transforms/CombineCooperativeMatrix/matmul-loop.lgc @@ -1,16 +1,17 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool lgc -; RUN: lgc -o - -passes=lgc-combine-cooperative-matrix %s | FileCheck --check-prefixes=CHECK %s +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool lgc --version 2 +; RUN: lgc -o - -passes='require,function(lgc-combine-cooperative-matrix)' %s | FileCheck --check-prefixes=CHECK %s define void @matmul_f16(ptr %ptr) { -; CHECK-LABEL: @matmul_f16( +; CHECK-LABEL: define void @matmul_f16 +; CHECK-SAME: (ptr [[PTR:%.*]]) { ; CHECK-NEXT: entry: -; CHECK-NEXT: [[ACCUM_LOAD:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[PTR:%.*]], i32 4, i1 false, i32 1, i32 1, i32 0) +; CHECK-NEXT: [[ACCUM_LOAD:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[PTR]], i32 4, i1 false, i32 1, i32 1, i32 0) ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: ; CHECK-NEXT: [[ACCUM_PHI:%.*]] = phi <8 x float> [ [[ACCUM_LOAD]], [[ENTRY:%.*]] ], [ [[MULADD:%.*]], [[LOOP]] ] ; CHECK-NEXT: [[A:%.*]] = call <8 x float> @getmat1() ; CHECK-NEXT: [[B:%.*]] = call <8 x float> @getmat1() -; CHECK-NEXT: [[MULADD]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[ACCUM_PHI]], i1 true, i1 true, i32 1, i32 1) +; CHECK-NEXT: [[MULADD]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[ACCUM_PHI]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) ; CHECK-NEXT: [[CC:%.*]] = call i1 @getcc() ; CHECK-NEXT: br i1 [[CC]], label [[LOOP]], label [[END:%.*]] ; CHECK: end: @@ -28,7 +29,7 @@ loop: %b = call <8 x float> @getmat1() %accum.cvt = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %accum.phi, i32 1, i32 1, i32 0, i32 1) - %muladd = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %accum.cvt, i1 true, i1 true, i32 1, i32 1) + %muladd = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %accum.cvt, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) %accum.next = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %muladd, i32 1, i32 1, i32 1, i32 0) %cc = call i1 @getcc() @@ -40,18 +41,19 @@ end: } define void @matmul_f16_initzero(ptr %ptr) { -; CHECK-LABEL: @matmul_f16_initzero( +; CHECK-LABEL: define void @matmul_f16_initzero +; CHECK-SAME: (ptr [[PTR:%.*]]) { ; CHECK-NEXT: entry: ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: ; CHECK-NEXT: [[ACCUM_PHI:%.*]] = phi <8 x float> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[MULADD:%.*]], [[LOOP]] ] ; CHECK-NEXT: [[A:%.*]] = call <8 x float> @getmat1() ; CHECK-NEXT: [[B:%.*]] = call <8 x float> @getmat1() -; CHECK-NEXT: [[MULADD]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[ACCUM_PHI]], i1 true, i1 true, i32 1, i32 1) +; CHECK-NEXT: [[MULADD]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[ACCUM_PHI]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) ; CHECK-NEXT: [[CC:%.*]] = call i1 @getcc() ; CHECK-NEXT: br i1 [[CC]], label [[LOOP]], label [[END:%.*]] ; CHECK: end: -; CHECK-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[PTR:%.*]], i32 4, i1 true, i32 1, i32 1, i32 0, <8 x float> [[MULADD]]) +; CHECK-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[PTR]], i32 4, i1 true, i32 1, i32 1, i32 0, <8 x float> [[MULADD]]) ; CHECK-NEXT: ret void ; entry: @@ -64,7 +66,7 @@ loop: %b = call <8 x float> @getmat1() %accum.cvt = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %accum.phi, i32 1, i32 1, i32 0, i32 1) - %muladd = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %accum.cvt, i1 true, i1 true, i32 1, i32 1) + %muladd = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %accum.cvt, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) %accum.next = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %muladd, i32 1, i32 1, i32 1, i32 0) %cc = call i1 @getcc() @@ -81,4 +83,4 @@ declare <8 x float> @getmat1() declare <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr, i32, i1, i32, i32, i32) declare <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32, <8 x float>, i32, i32, i32, i32) declare void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr, i32, i1, i32, i32, i32, <8 x float>) -declare <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i32.i32(<8 x float>, <8 x float>, <8 x float>, i1, i1, i32, i32) +declare <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float>, <8 x float>, <8 x float>, i1, i1, i1, i1, i32, i32) diff --git a/lgc/test/Transforms/CombineCooperativeMatrix/packed-accumulators.lgc b/lgc/test/Transforms/CombineCooperativeMatrix/packed-accumulators.lgc new file mode 100644 index 0000000000..cfe6d4d85f --- /dev/null +++ b/lgc/test/Transforms/CombineCooperativeMatrix/packed-accumulators.lgc @@ -0,0 +1,789 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool lgc --version 2 +; RUN: lgc --mcpu=gfx1100 -o - -passes='require,function(lgc-combine-cooperative-matrix)' %s | FileCheck --check-prefixes=GFX11 %s + +define void @matmul_f16_pack_simple(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f16_pack_simple +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0:[0-9]+]] +; GFX11-NEXT: [[MULADDLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[MULADDLO]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI]], i1 false) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP1]]) +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI]], i1 true) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP2]]) +; GFX11-NEXT: ret void +; +entry: + %muladdLo = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdHi = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %muladdLo) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %muladdHi) + ret void +} + +define void @matmul_f16_pack_chain_sequential(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f16_pack_chain_sequential +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[CHAIN0_1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CHAIN0_2:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[CHAIN0_1]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CHAIN1_1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[CHAIN0_2]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CHAIN1_2:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[CHAIN1_1]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN1_2]], i1 false) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP1]]) +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN1_2]], i1 true) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP2]]) +; GFX11-NEXT: ret void +; +entry: + %chain0.1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain0.2 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %chain0.1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain1.1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain1.2 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %chain1.1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %chain0.2) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %chain1.2) + ret void +} + +define void @matmul_f16_pack_chain_alternating(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f16_pack_chain_alternating +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[CHAIN0_1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CHAIN1_1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[CHAIN0_1]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CHAIN0_2:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[CHAIN1_1]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CHAIN1_2:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[CHAIN0_2]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN1_2]], i1 false) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP1]]) +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN1_2]], i1 true) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP2]]) +; GFX11-NEXT: ret void +; +entry: + %chain0.1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain1.1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain0.2 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %chain0.1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain1.2 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %chain1.1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %chain0.2) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %chain1.2) + ret void +} + +define void @matmul_f16_pack_chain_nested(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f16_pack_chain_nested +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[CHAIN0_1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CHAIN1_1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[CHAIN0_1]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CHAIN1_2:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[CHAIN1_1]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CHAIN0_2:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[CHAIN1_2]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN0_2]], i1 false) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP1]]) +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN0_2]], i1 true) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP2]]) +; GFX11-NEXT: ret void +; +entry: + %chain0.1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain1.1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain1.2 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %chain1.1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain0.2 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %chain0.1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %chain0.2) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %chain1.2) + ret void +} + +define void @matmul_f16_no_packable_chain(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0) { +; GFX11-LABEL: define void @matmul_f16_no_packable_chain +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[CHAIN0_1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[C0]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) +; GFX11-NEXT: [[CHAIN1_1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[CHAIN0_1]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[CHAIN1_1]]) +; GFX11-NEXT: ret void +; +entry: + %chain0.1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain1.1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %chain0.1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %chain1.1) + ret void +} + +define void @matmul_f16_chain_loop(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f16_chain_loop +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[CHAIN0_1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CHAIN1_1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[CHAIN0_1]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN1_1]], i1 false) #[[ATTR0]] +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN1_1]], i1 true) #[[ATTR0]] +; GFX11-NEXT: [[TMP3:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[TMP1]], <8 x float> [[TMP2]]) #[[ATTR0]] +; GFX11-NEXT: br label [[LOOP:%.*]] +; GFX11: loop: +; GFX11-NEXT: [[ACCUM1_PHI:%.*]] = phi <8 x float> [ [[TMP3]], [[ENTRY:%.*]] ], [ [[CHAIN1_2:%.*]], [[LOOP]] ] +; GFX11-NEXT: [[CHAIN0_2:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[ACCUM1_PHI]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CHAIN1_2]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[CHAIN0_2]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CC:%.*]] = call i1 @getcc() +; GFX11-NEXT: br i1 [[CC]], label [[LOOP]], label [[END:%.*]] +; GFX11: end: +; GFX11-NEXT: [[TMP4:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN1_2]], i1 false) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP4]]) +; GFX11-NEXT: [[TMP5:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN1_2]], i1 true) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP5]]) +; GFX11-NEXT: ret void +; +entry: + %chain0.1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain1.1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + br label %loop + +loop: + %accum0.phi = phi <8 x float> [ %chain0.1, %entry ], [ %chain0.2, %loop ] + %accum1.phi = phi <8 x float> [ %chain1.1, %entry ], [ %chain1.2, %loop ] + + %chain0.2 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %accum0.phi, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain1.2 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %accum1.phi, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + + %cc = call i1 @getcc() + br i1 %cc, label %loop, label %end + +end: + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %chain0.2) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %chain1.2) + ret void +} + +define void @matmul_f16_chain_loop_phis(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) {; GFX11-LABEL: define void @matmul_f16_chain_loop_phis +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[ACCUM0_LOAD:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[OUT0]], i32 4, i1 false, i32 1, i32 1, i32 0) +; GFX11-NEXT: [[ACCUM1_LOAD:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[OUT1]], i32 4, i1 false, i32 1, i32 1, i32 0) +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[ACCUM0_LOAD]], <8 x float> [[ACCUM1_LOAD]]) #[[ATTR0]] +; GFX11-NEXT: br label [[HEADER:%.*]] +; GFX11: header: +; GFX11-NEXT: [[ACCUM1_PHI:%.*]] = phi <8 x float> [ [[TMP0]], [[ENTRY:%.*]] ], [ [[MULADDHI:%.*]], [[LOOP:%.*]] ] +; GFX11-NEXT: [[CC:%.*]] = call i1 @getcc() +; GFX11-NEXT: br i1 [[CC]], label [[LOOP]], label [[END:%.*]] +; GFX11: loop: +; GFX11-NEXT: [[MULADDLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[ACCUM1_PHI]], i1 false, i1 false, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[MULADDLO]], i1 false, i1 false, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: br label [[HEADER]] +; GFX11: end: +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[ACCUM1_PHI]], i1 false) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 1, i32 0, <8 x float> [[TMP1]]) +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[ACCUM1_PHI]], i1 true) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 1, i32 0, <8 x float> [[TMP2]]) +; GFX11-NEXT: ret void +; +entry: + %accum0.load = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr %out0, i32 4, i1 false, i32 1, i32 0, i32 0) #0 + %accum1.load = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr %out1, i32 4, i1 false, i32 1, i32 0, i32 0) #0 + br label %header + +header: + %accum0.phi = phi <8 x float> [ %accum0.load, %entry ], [ %accum0.next, %loop ] + %accum1.phi = phi <8 x float> [ %accum1.load, %entry ], [ %accum1.next, %loop ] + %cc = call i1 @getcc() + br i1 %cc, label %loop, label %end + +loop: + %accum0.cvt = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %accum0.phi, i32 1, i32 1, i32 0, i32 1) + %accum1.cvt = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %accum1.phi, i32 1, i32 1, i32 0, i32 1) + %muladdLo = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %accum0.cvt, i1 false, i1 false, i1 false, i1 false, i32 1, i32 1) + %muladdHi = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %accum1.cvt, i1 false, i1 false, i1 false, i1 false, i32 1, i32 1) + %accum0.next = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %muladdLo, i32 1, i32 1, i32 1, i32 0) + %accum1.next = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %muladdHi, i32 1, i32 1, i32 1, i32 0) + + br label %header + +end: + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %accum0.phi) #2 + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %accum1.phi) #2 + ret void +} + +define void @matmul_f16_chain_branch(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f16_chain_branch +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[CHAIN0_1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CHAIN1_1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[CHAIN0_1]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CC:%.*]] = call i1 @getcc() +; GFX11-NEXT: br i1 [[CC]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]] +; GFX11: if_true: +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN1_1]], i1 false) #[[ATTR0]] +; GFX11-NEXT: [[CHAIN0_2:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP1]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN1_1]], i1 true) #[[ATTR0]] +; GFX11-NEXT: [[CHAIN1_2:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP2]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) +; GFX11-NEXT: br label [[END:%.*]] +; GFX11: if_false: +; GFX11-NEXT: [[A_FALSE:%.*]] = call <8 x float> @getmat1() +; GFX11-NEXT: [[B_FALSE:%.*]] = call <8 x float> @getmat1() +; GFX11-NEXT: [[TMP3:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN1_1]], i1 false) #[[ATTR0]] +; GFX11-NEXT: [[CHAIN0_3:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A_FALSE]], <8 x float> [[B_FALSE]], <8 x float> [[TMP3]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) +; GFX11-NEXT: [[TMP4:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN1_1]], i1 true) #[[ATTR0]] +; GFX11-NEXT: [[CHAIN1_3:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A_FALSE]], <8 x float> [[B_FALSE]], <8 x float> [[TMP4]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) +; GFX11-NEXT: br label [[END]] +; GFX11: end: +; GFX11-NEXT: [[ACCUM0_PHI:%.*]] = phi <8 x float> [ [[CHAIN0_2]], [[IF_TRUE]] ], [ [[CHAIN0_3]], [[IF_FALSE]] ] +; GFX11-NEXT: [[ACCUM1_PHI:%.*]] = phi <8 x float> [ [[CHAIN1_2]], [[IF_TRUE]] ], [ [[CHAIN1_3]], [[IF_FALSE]] ] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[ACCUM0_PHI]]) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[ACCUM1_PHI]]) +; GFX11-NEXT: ret void +; +entry: + %chain0.1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain1.1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %cc = call i1 @getcc() + br i1 %cc, label %if_true, label %if_false + +if_true: + %chain0.2 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %chain0.1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain1.2 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %chain1.1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + + br label %end +if_false: + %a.false = call <8 x float> @getmat1() + %b.false = call <8 x float> @getmat1() + + %chain0.3 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a.false, <8 x float> %b.false, <8 x float> %chain0.1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain1.3 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a.false, <8 x float> %b.false, <8 x float> %chain1.1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + + br label %end + +end: + %accum0.phi = phi <8 x float> [ %chain0.2, %if_true ], [ %chain0.3, %if_false ] + %accum1.phi = phi <8 x float> [ %chain1.2, %if_true ], [ %chain1.3, %if_false ] + + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %accum0.phi) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %accum1.phi) + ret void +} + +define void @matmul_f16_chain_diff_bbs(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f16_chain_diff_bbs +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[CHAIN0_1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CHAIN1_1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[CHAIN0_1]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CC:%.*]] = call i1 @getcc() +; GFX11-NEXT: br label [[CONT:%.*]] +; GFX11: cont: +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN1_1]], i1 false) #[[ATTR0]] +; GFX11-NEXT: [[CHAIN0_2:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP1]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN1_1]], i1 true) #[[ATTR0]] +; GFX11-NEXT: [[CHAIN1_2:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP2]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) +; GFX11-NEXT: br label [[END:%.*]] +; GFX11: end: +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[CHAIN0_2]]) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[CHAIN1_2]]) +; GFX11-NEXT: ret void +; +entry: + %chain0.1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain1.1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %cc = call i1 @getcc() + br label %cont +cont: + %chain0.2 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %chain0.1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain1.2 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %chain1.1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + + br label %end +end: + + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %chain0.2) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %chain1.2) + ret void +} + +define void @matmul_f16_pack_loop(ptr %out0, ptr %out1) { +; GFX11-LABEL: define void @matmul_f16_pack_loop +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[ACCUM0_LOAD:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[OUT0]], i32 4, i1 false, i32 1, i32 1, i32 0) +; GFX11-NEXT: [[ACCUM1_LOAD:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[OUT1]], i32 4, i1 false, i32 1, i32 1, i32 0) +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[ACCUM0_LOAD]], <8 x float> [[ACCUM1_LOAD]]) #[[ATTR0]] +; GFX11-NEXT: br label [[LOOP:%.*]] +; GFX11: loop: +; GFX11-NEXT: [[ACCUM1_PHI:%.*]] = phi <8 x float> [ [[TMP0]], [[ENTRY:%.*]] ], [ [[MULADDHI:%.*]], [[LOOP]] ] +; GFX11-NEXT: [[A:%.*]] = call <8 x float> @getmat1() +; GFX11-NEXT: [[B:%.*]] = call <8 x float> @getmat1() +; GFX11-NEXT: [[MULADDLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[ACCUM1_PHI]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[MULADDLO]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CC:%.*]] = call i1 @getcc() +; GFX11-NEXT: br i1 [[CC]], label [[LOOP]], label [[END:%.*]] +; GFX11: end: +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI]], i1 false) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 1, i32 0, <8 x float> [[TMP1]]) +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI]], i1 true) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 1, i32 0, <8 x float> [[TMP2]]) +; GFX11-NEXT: ret void +; +entry: + %accum0.load = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr %out0, i32 4, i1 false, i32 1, i32 0, i32 0) + %accum1.load = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr %out1, i32 4, i1 false, i32 1, i32 0, i32 0) + br label %loop + +loop: + %accum0.phi = phi <8 x float> [ %accum0.load, %entry ], [ %accum0.next, %loop ] + %accum1.phi = phi <8 x float> [ %accum1.load, %entry ], [ %accum1.next, %loop ] + + %a = call <8 x float> @getmat1() + %b = call <8 x float> @getmat1() + + %accum0.cvt = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %accum0.phi, i32 1, i32 1, i32 0, i32 1) + %accum1.cvt = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %accum1.phi, i32 1, i32 1, i32 0, i32 1) + %muladdLo = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %accum0.cvt, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdHi = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %accum1.cvt, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %accum0.next = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %muladdLo, i32 1, i32 1, i32 1, i32 0) + %accum1.next = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %muladdHi, i32 1, i32 1, i32 1, i32 0) + + %cc = call i1 @getcc() + br i1 %cc, label %loop, label %end + +end: + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %accum0.next) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %accum1.next) + ret void +} + +define void @matmul_f16_pack_scalar_same(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f16_pack_scalar_same +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[MULADDLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[MULADDLO]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.v2f16.i32.i32(<8 x float> [[MULADDHI]], <2 x half> , i32 6, i32 1) #[[ATTR1:[0-9]+]] +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[TMP1]], i1 false) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP2]]) +; GFX11-NEXT: [[TMP3:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[TMP1]], i1 true) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP3]]) +; GFX11-NEXT: ret void +; +entry: + %muladdLo = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdHi = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %scaledLo = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %muladdLo, half 0xH310F, i32 1, i32 1) + %scaledHi = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %muladdHi, half 0xH310F, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %scaledLo) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %scaledHi) + ret void +} + +define void @matmul_f16_pack_scalar_different(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f16_pack_scalar_different +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[MULADDLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[MULADDLO]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.v2f16.i32.i32(<8 x float> [[MULADDHI]], <2 x half> , i32 6, i32 1) #[[ATTR1]] +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[TMP1]], i1 false) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP2]]) +; GFX11-NEXT: [[TMP3:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[TMP1]], i1 true) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP3]]) +; GFX11-NEXT: ret void +; +entry: + %muladdLo = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdHi = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %scaledLo = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %muladdLo, half 0xH310F, i32 1, i32 1) + %scaledHi = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %muladdHi, half 0xH3100, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %scaledLo) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %scaledHi) + ret void +} + +define void @matmul_f16_pack_scalar_only_lo(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f16_pack_scalar_only_lo +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[MULADDLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[MULADDLO]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI]], i1 false) #[[ATTR0]] +; GFX11-NEXT: [[SCALEDLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> [[TMP1]], half 0xH310F, i32 1, i32 1) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[SCALEDLO]]) +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI]], i1 true) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP2]]) +; GFX11-NEXT: ret void +; +entry: + %muladdLo = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdHi = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %scaledLo = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %muladdLo, half 0xH310F, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %scaledLo) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %muladdHi) + ret void +} + +define void @matmul_f16_pack_scalar_only_hi(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f16_pack_scalar_only_hi +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[MULADDLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[MULADDLO]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI]], i1 true) #[[ATTR0]] +; GFX11-NEXT: [[SCALEDHI:%.*]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> [[TMP1]], half 0xH3100, i32 1, i32 1) +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI]], i1 false) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP2]]) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[SCALEDHI]]) +; GFX11-NEXT: ret void +; +entry: + %muladdLo = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdHi = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %scaledHi = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %muladdHi, half 0xH3100, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %muladdLo) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %scaledHi) + ret void +} + +define void @matmul_f16_pack_scalar_diff_bbs(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f16_pack_scalar_diff_bbs +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[MULADDLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[MULADDLO]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: br label [[SCALE_LO:%.*]] +; GFX11: scale_lo: +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI]], i1 false) #[[ATTR0]] +; GFX11-NEXT: [[SCALEDLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> [[TMP1]], half 0xH310F, i32 1, i32 1) +; GFX11-NEXT: br label [[SCALE_HI:%.*]] +; GFX11: scale_hi: +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI]], i1 true) #[[ATTR0]] +; GFX11-NEXT: [[SCALEDHI:%.*]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> [[TMP2]], half 0xH310F, i32 1, i32 1) +; GFX11-NEXT: br label [[END:%.*]] +; GFX11: end: +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[SCALEDLO]]) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[SCALEDHI]]) +; GFX11-NEXT: ret void +; +entry: + %muladdLo = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdHi = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + br label %scale_lo + +scale_lo: + %scaledLo = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %muladdLo, half 0xH310F, i32 1, i32 1) + br label %scale_hi + +scale_hi: + %scaledHi = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %muladdHi, half 0xH310F, i32 1, i32 1) + br label %end + +end: + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %scaledLo) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %scaledHi) + ret void +} + +define void @matmul_f16_pack_user_between_scalar(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f16_pack_user_between_scalar +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[MULADDLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[MULADDLO]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: br label [[SCALE:%.*]] +; GFX11: scale: +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI]], i1 false) #[[ATTR0]] +; GFX11-NEXT: [[SCALEDLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> [[TMP1]], half 0xH310F, i32 1, i32 1) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[SCALEDLO]]) +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI]], i1 true) #[[ATTR0]] +; GFX11-NEXT: [[SCALEDHI:%.*]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> [[TMP2]], half 0xH310F, i32 1, i32 1) +; GFX11-NEXT: br label [[END:%.*]] +; GFX11: end: +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[SCALEDLO]]) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[SCALEDHI]]) +; GFX11-NEXT: ret void +; +entry: + %muladdLo = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdHi = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + br label %scale + +scale: + %scaledLo = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %muladdLo, half 0xH310F, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %scaledLo) + %scaledHi = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %muladdHi, half 0xH310F, i32 1, i32 1) + br label %end + +end: + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %scaledLo) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %scaledHi) + ret void +} + +define void @matmul_f16_pack_factor_between_scalar(ptr %in, ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f16_pack_factor_between_scalar +; GFX11-SAME: (ptr [[IN:%.*]], ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[MULADDLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[MULADDLO]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: br label [[SCALE:%.*]] +; GFX11: scale: +; GFX11-NEXT: [[FACTORHI:%.*]] = load half, ptr [[IN]], align 2 +; GFX11-NEXT: [[TMP1:%.*]] = insertelement <2 x half> , half [[FACTORHI]], i32 1 +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.v2f16.i32.i32(<8 x float> [[MULADDHI]], <2 x half> [[TMP1]], i32 6, i32 1) #[[ATTR1]] +; GFX11-NEXT: br label [[END:%.*]] +; GFX11: end: +; GFX11-NEXT: [[TMP3:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[TMP2]], i1 false) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP3]]) +; GFX11-NEXT: [[TMP4:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[TMP2]], i1 true) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP4]]) +; GFX11-NEXT: ret void +; +entry: + %muladdLo = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdHi = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + br label %scale + +scale: + %scaledLo = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %muladdLo, half 0xH310F, i32 1, i32 1) + %factorHi = load half, ptr %in + %scaledHi = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %muladdHi, half %factorHi, i32 1, i32 1) + br label %end + +end: + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %scaledLo) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %scaledHi) + ret void +} + +define void @matmul_f16_pack_binop_fadd(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1, <8 x float> %c2, <8 x float> %c3) { +; GFX11-LABEL: define void @matmul_f16_pack_binop_fadd +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]], <8 x float> [[C2:%.*]], <8 x float> [[C3:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[MULADDLO0:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI0:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[MULADDLO0]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C2]], <8 x float> [[C3]]) #[[ATTR0]] +; GFX11-NEXT: [[MULADDLO1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP1]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[MULADDLO1]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI0]], i1 false) #[[ATTR0]] +; GFX11-NEXT: [[TMP3:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI1]], i1 false) #[[ATTR0]] +; GFX11-NEXT: [[BINOPLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> [[TMP2]], <8 x float> [[TMP3]], i32 1, i32 1) +; GFX11-NEXT: [[TMP4:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI0]], i1 true) #[[ATTR0]] +; GFX11-NEXT: [[TMP5:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI1]], i1 true) #[[ATTR0]] +; GFX11-NEXT: [[BINOPHI:%.*]] = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> [[TMP4]], <8 x float> [[TMP5]], i32 1, i32 1) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[BINOPLO]]) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[BINOPHI]]) +; GFX11-NEXT: ret void +; +entry: + %muladdLo0 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdHi0 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdLo1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c2, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdHi1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c3, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %binOpLo = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> %muladdLo0, <8 x float> %muladdLo1, i32 1, i32 1) #3 + %binOpHi = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> %muladdHi0, <8 x float> %muladdHi1, i32 1, i32 1) #3 + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %binOpLo) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %binOpHi) + ret void +} + +define void @matmul_f16_pack_binop_incompatible_matrices(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1, <8 x float> %c2, <8 x float> %c3) { +; GFX11-LABEL: define void @matmul_f16_pack_binop_incompatible_matrices +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]], <8 x float> [[C2:%.*]], <8 x float> [[C3:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[MULADDLO0:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI0:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[MULADDLO0]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C2]], <8 x float> [[C3]]) #[[ATTR0]] +; GFX11-NEXT: [[MULADDLO1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP1]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[MULADDLO1]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI0]], i1 false) #[[ATTR0]] +; GFX11-NEXT: [[TMP3:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI1]], i1 false) #[[ATTR0]] +; GFX11-NEXT: [[BINOPLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> [[TMP2]], <8 x float> [[TMP3]], i32 1, i32 1) +; GFX11-NEXT: [[TMP4:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI0]], i1 true) #[[ATTR0]] +; GFX11-NEXT: [[TMP5:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI0]], i1 true) #[[ATTR0]] +; GFX11-NEXT: [[BINOPHI:%.*]] = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> [[TMP5]], <8 x float> [[TMP4]], i32 1, i32 1) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[BINOPLO]]) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[BINOPHI]]) +; GFX11-NEXT: ret void +; +entry: + %muladdLo0 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdHi0 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdLo1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c2, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdHi1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c3, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %binOpLo = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> %muladdLo0, <8 x float> %muladdLo1, i32 1, i32 1) #3 + %binOpHi = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> %muladdHi0, <8 x float> %muladdHi0, i32 1, i32 1) #3 + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %binOpLo) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %binOpHi) + ret void +} + +define void @matmul_f16_pack_binop_incompatible_arithop(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1, <8 x float> %c2, <8 x float> %c3) { +; GFX11-LABEL: define void @matmul_f16_pack_binop_incompatible_arithop +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]], <8 x float> [[C2:%.*]], <8 x float> [[C3:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[MULADDLO0:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI0:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[MULADDLO0]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C2]], <8 x float> [[C3]]) #[[ATTR0]] +; GFX11-NEXT: [[MULADDLO1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP1]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[MULADDLO1]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI0]], i1 false) #[[ATTR0]] +; GFX11-NEXT: [[TMP3:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI1]], i1 false) #[[ATTR0]] +; GFX11-NEXT: [[BINOPLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> [[TMP2]], <8 x float> [[TMP3]], i32 1, i32 1) +; GFX11-NEXT: [[TMP4:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI0]], i1 true) #[[ATTR0]] +; GFX11-NEXT: [[TMP5:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI1]], i1 true) #[[ATTR0]] +; GFX11-NEXT: [[BINOPHI:%.*]] = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 3, <8 x float> [[TMP4]], <8 x float> [[TMP5]], i32 1, i32 1) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[BINOPLO]]) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[BINOPHI]]) +; GFX11-NEXT: ret void +; +entry: + %muladdLo0 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdHi0 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdLo1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c2, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdHi1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c3, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %binOpLo = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> %muladdLo0, <8 x float> %muladdLo1, i32 1, i32 1) #3 + %binOpHi = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 3, <8 x float> %muladdHi0, <8 x float> %muladdHi1, i32 1, i32 1) #3 + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %binOpLo) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %binOpHi) + ret void +} + +define void @matmul_f16_unpack_before_convert(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f16_unpack_before_convert +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[MULADDLO0:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI0:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[MULADDLO0]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI0]], i1 false) #[[ATTR0]] +; GFX11-NEXT: [[CONVERTLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> [[TMP1]], i32 1, i32 1, i32 1, i32 0) +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI0]], i1 true) #[[ATTR0]] +; GFX11-NEXT: [[CONVERTHI:%.*]] = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> [[TMP2]], i32 1, i32 1, i32 1, i32 0) +; GFX11-NEXT: [[TMP3:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[MULADDLO1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[CONVERTLO]], <8 x float> [[B]], <8 x float> [[TMP3]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[CONVERTHI]], <8 x float> [[MULADDLO1]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP4:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI1]], i1 false) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP4]]) +; GFX11-NEXT: [[TMP5:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[MULADDHI1]], i1 true) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP5]]) +; GFX11-NEXT: ret void +; +entry: + %muladdLo0 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdHi0 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %convertLo = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %muladdLo0, i32 1, i32 1, i32 1, i32 0) + %convertHi = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %muladdHi0, i32 1, i32 1, i32 1, i32 0) + %muladdLo1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %convertLo, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %muladdHi1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %convertHi, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %muladdLo1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %muladdHi1) + ret void +} + +define void @matmul_f32_no_pack(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f32_no_pack +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[MULADDLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[C0]], i1 true, i1 true, i1 false, i1 false, i32 2, i32 2) +; GFX11-NEXT: [[MULADDHI:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[C1]], i1 true, i1 true, i1 false, i1 false, i32 2, i32 2) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 2, i32 0, i32 0, <8 x float> [[MULADDLO]]) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 2, i32 0, i32 0, <8 x float> [[MULADDHI]]) +; GFX11-NEXT: ret void +; +entry: + %muladdLo = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 2, i32 2) + %muladdHi = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 2, i32 2) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 2, i32 0, i32 0, <8 x float> %muladdLo) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 2, i32 0, i32 0, <8 x float> %muladdHi) + ret void +} + +define void @matmul_f16_modified_accumulator(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f16_modified_accumulator +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[MULADDLO:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[C0]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) +; GFX11-NEXT: [[ACCUM_C2:%.*]] = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> [[MULADDLO]], <8 x float> [[C1]], i32 1, i32 1) +; GFX11-NEXT: [[MULADDHI:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[ACCUM_C2]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[MULADDLO]]) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[MULADDHI]]) +; GFX11-NEXT: ret void +; +entry: + %muladdLo = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %accum.c2 = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> %muladdLo, <8 x float> %c1, i32 1, i32 1) + %muladdHi = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %accum.c2, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %muladdLo) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %muladdHi) + ret void +} + +define void @matmul_f16_store_between_muladds(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f16_store_between_muladds +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[CHAIN0_1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CHAIN1_1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[CHAIN0_1]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN1_1]], i1 true) #[[ATTR0]] +; GFX11-NEXT: [[CHAIN0_2:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP1]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[CHAIN0_2]]) +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN1_1]], i1 true) #[[ATTR0]] +; GFX11-NEXT: [[CHAIN1_2:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP2]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[CHAIN1_2]]) +; GFX11-NEXT: ret void +; +entry: + %chain0.1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain1.1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain0.2 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %chain1.1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %chain0.2) + %chain1.2 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %chain1.1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %chain1.2) + ret void +} + +define void @matmul_f16_store_within_chain(ptr %out0, ptr %out1, <8 x float> %a, <8 x float> %b, <8 x float> %c0, <8 x float> %c1) { +; GFX11-LABEL: define void @matmul_f16_store_within_chain +; GFX11-SAME: (ptr [[OUT0:%.*]], ptr [[OUT1:%.*]], <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x float> [[C0:%.*]], <8 x float> [[C1:%.*]]) { +; GFX11-NEXT: entry: +; GFX11-NEXT: [[TMP0:%.*]] = call <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32.v8f32(<8 x float> [[C0]], <8 x float> [[C1]]) #[[ATTR0]] +; GFX11-NEXT: [[CHAIN0_1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP0]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CHAIN1_1:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[CHAIN0_1]], i1 true, i1 true, i1 true, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[CHAIN0_2:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[CHAIN1_1]], i1 true, i1 true, i1 false, i1 true, i32 1, i32 1) +; GFX11-NEXT: [[TMP1:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN0_2]], i1 false) #[[ATTR0]] +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[TMP1]]) +; GFX11-NEXT: [[TMP2:%.*]] = call <8 x float> @lgc.cooperative.matrix.unpack.v8f32.v8f32.i1(<8 x float> [[CHAIN0_2]], i1 true) #[[ATTR0]] +; GFX11-NEXT: [[CHAIN1_2:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[A]], <8 x float> [[B]], <8 x float> [[TMP2]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) +; GFX11-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[OUT1]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[CHAIN1_2]]) +; GFX11-NEXT: ret void +; +entry: + %chain0.1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c0, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain1.1 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %c1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %chain0.2 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %chain0.1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %chain0.2) + %chain1.2 = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %a, <8 x float> %b, <8 x float> %chain1.1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %out1, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %chain1.2) + ret void +} + +declare i1 @getcc() +declare <8 x float> @getmat1() + +declare <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr, i32, i1, i32, i32, i32) +declare <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32, <8 x float>, i32, i32, i32, i32) +declare void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr, i32, i1, i32, i32, i32, <8 x float>) +declare <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float>, <8 x float>, <8 x float>, i1, i1, i1, i1, i32, i32) +declare <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float>, half, i32, i32) +declare <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32, <8 x float>, <8 x float>, i32, i32) diff --git a/lgc/test/Transforms/CombineCooperativeMatrix/simple.lgc b/lgc/test/Transforms/CombineCooperativeMatrix/simple.lgc index e1570be25e..117600ccc8 100644 --- a/lgc/test/Transforms/CombineCooperativeMatrix/simple.lgc +++ b/lgc/test/Transforms/CombineCooperativeMatrix/simple.lgc @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool lgc -; RUN: lgc -o - -passes=lgc-combine-cooperative-matrix %s | FileCheck --check-prefixes=CHECK %s +; RUN: lgc -o - -passes='require,function(lgc-combine-cooperative-matrix)' %s | FileCheck --check-prefixes=CHECK %s define <8 x float> @noop_transpose(<8 x float> %x) { ; CHECK-LABEL: @noop_transpose( diff --git a/lgc/test/Transforms/CombineCooperativeMatrix/unhandled-inout.lgc b/lgc/test/Transforms/CombineCooperativeMatrix/unhandled-inout.lgc index 6bc9523ab9..08f079cb89 100644 --- a/lgc/test/Transforms/CombineCooperativeMatrix/unhandled-inout.lgc +++ b/lgc/test/Transforms/CombineCooperativeMatrix/unhandled-inout.lgc @@ -1,15 +1,16 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool lgc -; RUN: lgc -o - -passes=lgc-combine-cooperative-matrix %s | FileCheck --check-prefixes=CHECK %s +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool lgc --version 2 +; RUN: lgc -o - -passes='require,function(lgc-combine-cooperative-matrix)' %s | FileCheck --check-prefixes=CHECK %s define <8 x float> @insert_transpose(<8 x float> %x) { -; CHECK-LABEL: @insert_transpose( +; CHECK-LABEL: define <8 x float> @insert_transpose +; CHECK-SAME: (<8 x float> [[X:%.*]]) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[GUARD:%.*]] = call i1 @getcc() ; CHECK-NEXT: br i1 [[GUARD]], label [[LOOP:%.*]], label [[END:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[V_LOOP:%.*]] = phi <8 x float> [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[MULADD:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[V_LOOP:%.*]] = phi <8 x float> [ [[X]], [[ENTRY:%.*]] ], [ [[MULADD:%.*]], [[LOOP]] ] ; CHECK-NEXT: [[F:%.*]] = call <8 x float> @getmat1() -; CHECK-NEXT: [[MULADD]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i32.i32(<8 x float> [[F]], <8 x float> [[F]], <8 x float> [[V_LOOP]], i1 true, i1 true, i32 1, i32 1) +; CHECK-NEXT: [[MULADD]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[F]], <8 x float> [[F]], <8 x float> [[V_LOOP]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) ; CHECK-NEXT: [[CC:%.*]] = call i1 @getcc() ; CHECK-NEXT: br i1 [[CC]], label [[LOOP]], label [[END]] ; CHECK: end: @@ -27,7 +28,7 @@ loop: %f = call <8 x float> @getmat1() %pre.t = call <8 x float> @lgc.cooperative.matrix.transpose.v8f32.v8f32.i32.i32(<8 x float> %v.loop, i32 1, i32 0) - %muladd = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i32.i32(<8 x float> %f, <8 x float> %f, <8 x float> %pre.t, i1 true, i1 true, i32 1, i32 1) + %muladd = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %f, <8 x float> %f, <8 x float> %pre.t, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) %v.next = call <8 x float> @lgc.cooperative.matrix.transpose.v8f32.v8f32.i32.i32(<8 x float> %muladd, i32 1, i32 0) %cc = call i1 @getcc() @@ -39,28 +40,30 @@ end: } define <8 x float> @reuse_transpose(<8 x float> %x) { -; CHECK-LABEL: @reuse_transpose( -; CHECK-NEXT: [[T1:%.*]] = call <8 x float> @lgc.cooperative.matrix.transpose.v8f32.v8f32.i32.i32(<8 x float> [[X:%.*]], i32 1, i32 0) -; CHECK-NEXT: [[R:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i32.i32(<8 x float> [[T1]], <8 x float> [[X]], <8 x float> zeroinitializer, i1 true, i1 true, i32 1, i32 1) +; CHECK-LABEL: define <8 x float> @reuse_transpose +; CHECK-SAME: (<8 x float> [[X:%.*]]) { +; CHECK-NEXT: [[T1:%.*]] = call <8 x float> @lgc.cooperative.matrix.transpose.v8f32.v8f32.i32.i32(<8 x float> [[X]], i32 1, i32 0) +; CHECK-NEXT: [[R:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[T1]], <8 x float> [[X]], <8 x float> zeroinitializer, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) ; CHECK-NEXT: ret <8 x float> [[R]] ; %t1 = call <8 x float> @lgc.cooperative.matrix.transpose.v8f32.v8f32.i32.i32(<8 x float> %x, i32 1, i32 0) %t2 = call <8 x float> @lgc.cooperative.matrix.transpose.v8f32.v8f32.i32.i32(<8 x float> %t1, i32 1, i32 0) - %r = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i32.i32(<8 x float> %t1, <8 x float> %t2, <8 x float> zeroinitializer, i1 true, i1 true, i32 1, i32 1) + %r = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %t1, <8 x float> %t2, <8 x float> zeroinitializer, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) ret <8 x float> %r } ; NOTE: This test leaves a convert inside the loop. Rely on other passes to sink it out. define <8 x float> @insert_convert(ptr %ptr) { -; CHECK-LABEL: @insert_convert( +; CHECK-LABEL: define <8 x float> @insert_convert +; CHECK-SAME: (ptr [[PTR:%.*]]) { ; CHECK-NEXT: entry: -; CHECK-NEXT: [[LOAD:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[PTR:%.*]], i32 4, i1 false, i32 1, i32 1, i32 0) +; CHECK-NEXT: [[LOAD:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[PTR]], i32 4, i1 false, i32 1, i32 1, i32 0) ; CHECK-NEXT: [[GUARD:%.*]] = call i1 @getcc() ; CHECK-NEXT: br i1 [[GUARD]], label [[LOOP:%.*]], label [[END:%.*]] ; CHECK: loop: ; CHECK-NEXT: [[V_LOOP:%.*]] = phi <8 x float> [ [[LOAD]], [[ENTRY:%.*]] ], [ [[MULADD:%.*]], [[LOOP]] ] ; CHECK-NEXT: [[F:%.*]] = call <8 x float> @getmat1() -; CHECK-NEXT: [[MULADD]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i32.i32(<8 x float> [[F]], <8 x float> [[F]], <8 x float> [[V_LOOP]], i1 true, i1 true, i32 1, i32 1) +; CHECK-NEXT: [[MULADD]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[F]], <8 x float> [[F]], <8 x float> [[V_LOOP]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) ; CHECK-NEXT: [[CC:%.*]] = call i1 @getcc() ; CHECK-NEXT: br i1 [[CC]], label [[LOOP]], label [[END]] ; CHECK: end: @@ -78,7 +81,7 @@ loop: %f = call <8 x float> @getmat1() %pre = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %v.loop, i32 1, i32 1, i32 0, i32 1) - %muladd = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i32.i32(<8 x float> %f, <8 x float> %f, <8 x float> %pre, i1 true, i1 true, i32 1, i32 1) + %muladd = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %f, <8 x float> %f, <8 x float> %pre, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) %v.next = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %muladd, i32 1, i32 1, i32 1, i32 0) %cc = call i1 @getcc() @@ -90,21 +93,309 @@ end: } define <8 x float> @reuse_convert(<8 x float> %x) { -; CHECK-LABEL: @reuse_convert( -; CHECK-NEXT: [[CVT1:%.*]] = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> [[X:%.*]], i32 1, i32 1, i32 0, i32 1) -; CHECK-NEXT: [[R:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i32.i32(<8 x float> [[X]], <8 x float> [[X]], <8 x float> [[CVT1]], i1 true, i1 true, i32 1, i32 1) +; CHECK-LABEL: define <8 x float> @reuse_convert +; CHECK-SAME: (<8 x float> [[X:%.*]]) { +; CHECK-NEXT: [[CVT1:%.*]] = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> [[X]], i32 1, i32 1, i32 0, i32 1) +; CHECK-NEXT: [[R:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[X]], <8 x float> [[X]], <8 x float> [[CVT1]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) ; CHECK-NEXT: ret <8 x float> [[R]] ; %cvt1 = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %x, i32 1, i32 1, i32 0, i32 1) %cvt2 = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %cvt1, i32 1, i32 1, i32 1, i32 0) - %r = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i32.i32(<8 x float> %cvt2, <8 x float> %cvt2, <8 x float> %cvt1, i1 true, i1 true, i32 1, i32 1) + %r = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %cvt2, <8 x float> %cvt2, <8 x float> %cvt1, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) ret <8 x float> %r } +define void @convert_to_acc_inner_binop(ptr %ptr0, ptr %ptr1) { +; CHECK-LABEL: define void @convert_to_acc_inner_binop +; CHECK-SAME: (ptr [[PTR0:%.*]], ptr [[PTR1:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LOAD_A:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[PTR0]], i32 4, i1 false, i32 1, i32 0, i32 0) +; CHECK-NEXT: [[LOAD_B:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[PTR1]], i32 4, i1 false, i32 1, i32 0, i32 0) +; CHECK-NEXT: [[BINOP:%.*]] = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> [[LOAD_A]], <8 x float> [[LOAD_B]], i32 1, i32 0) +; CHECK-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[PTR0]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[BINOP]]) +; CHECK-NEXT: ret void +; +entry: + %load.a = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr %ptr0, i32 4, i1 false, i32 1, i32 0, i32 0) + %load.b = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr %ptr1, i32 4, i1 false, i32 1, i32 0, i32 0) + %conv.a = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %load.a, i32 1, i32 1, i32 0, i32 1) + %conv.b = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %load.b, i32 1, i32 1, i32 0, i32 1) + %binop = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> %conv.a, <8 x float> %conv.b, i32 1, i32 1) + %conv.post = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %binop, i32 1, i32 1, i32 1, i32 0) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %ptr0, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %conv.post) + ret void +} + +define void @convert_to_acc_inner_times_scalar(ptr %ptr) { +; CHECK-LABEL: define void @convert_to_acc_inner_times_scalar +; CHECK-SAME: (ptr [[PTR:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LOAD:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[PTR]], i32 4, i1 false, i32 1, i32 0, i32 0) +; CHECK-NEXT: [[SCALAR:%.*]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> [[LOAD]], half 0xH310F, i32 1, i32 0) +; CHECK-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[PTR]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[SCALAR]]) +; CHECK-NEXT: ret void +; +entry: + %load = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr %ptr, i32 4, i1 false, i32 1, i32 0, i32 0) + %conv.pre = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %load, i32 1, i32 1, i32 0, i32 1) + %scalar = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %conv.pre, half 0xH310F, i32 1, i32 1) + %conv.post = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %scalar, i32 1, i32 1, i32 1, i32 0) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %ptr, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %conv.post) + ret void +} + +define void @convert_to_fact_inner_binop(ptr %ptr0, ptr %ptr1) { +; CHECK-LABEL: define void @convert_to_fact_inner_binop +; CHECK-SAME: (ptr [[PTR0:%.*]], ptr [[PTR1:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LOAD_A:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[PTR0]], i32 4, i1 false, i32 1, i32 1, i32 0) +; CHECK-NEXT: [[LOAD_B:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[PTR1]], i32 4, i1 false, i32 1, i32 1, i32 0) +; CHECK-NEXT: [[BINOP:%.*]] = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> [[LOAD_A]], <8 x float> [[LOAD_B]], i32 1, i32 1) +; CHECK-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[PTR0]], i32 4, i1 true, i32 1, i32 1, i32 0, <8 x float> [[BINOP]]) +; CHECK-NEXT: ret void +; +entry: + %load.a = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr %ptr0, i32 4, i1 false, i32 1, i32 1, i32 0) + %load.b = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr %ptr1, i32 4, i1 false, i32 1, i32 1, i32 0) + %conv.a = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %load.a, i32 1, i32 1, i32 1, i32 0) + %conv.b = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %load.b, i32 1, i32 1, i32 1, i32 0) + %binop = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> %conv.a, <8 x float> %conv.b, i32 1, i32 0) + %conv.post = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %binop, i32 1, i32 1, i32 0, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %ptr0, i32 4, i1 true, i32 1, i32 1, i32 0, <8 x float> %conv.post) + ret void +} + +define void @convert_to_fact_inner_times_scalar(ptr %ptr) { +; CHECK-LABEL: define void @convert_to_fact_inner_times_scalar +; CHECK-SAME: (ptr [[PTR:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LOAD:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[PTR]], i32 4, i1 false, i32 1, i32 1, i32 0) +; CHECK-NEXT: [[SCALAR:%.*]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> [[LOAD]], half 0xH310F, i32 1, i32 1) +; CHECK-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[PTR]], i32 4, i1 true, i32 1, i32 1, i32 0, <8 x float> [[SCALAR]]) +; CHECK-NEXT: ret void +; +entry: + %load = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr %ptr, i32 4, i1 false, i32 1, i32 1, i32 0) + %conv.pre = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %load, i32 1, i32 1, i32 1, i32 0) + %scalar = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %conv.pre, half 0xH310F, i32 1, i32 0) + %conv.post = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %scalar, i32 1, i32 1, i32 0, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %ptr, i32 4, i1 true, i32 1, i32 1, i32 0, <8 x float> %conv.post) + ret void +} + +define void @convert_to_acc_inner_chain(ptr %ptr) { +; CHECK-LABEL: define void @convert_to_acc_inner_chain +; CHECK-SAME: (ptr [[PTR:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LOAD:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[PTR]], i32 4, i1 false, i32 1, i32 1, i32 0) +; CHECK-NEXT: [[GUARD:%.*]] = call i1 @getcc() +; CHECK-NEXT: br i1 [[GUARD]], label [[LOOP:%.*]], label [[END:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[V_LOOP:%.*]] = phi <8 x float> [ [[LOAD]], [[ENTRY:%.*]] ], [ [[SCALAR:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[F:%.*]] = call <8 x float> @getmat1() +; CHECK-NEXT: [[MULADD:%.*]] = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> [[F]], <8 x float> [[F]], <8 x float> [[V_LOOP]], i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) +; CHECK-NEXT: [[BINOP:%.*]] = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> [[MULADD]], <8 x float> [[MULADD]], i32 1, i32 1) +; CHECK-NEXT: [[SCALAR]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> [[BINOP]], half 0xH310F, i32 1, i32 1) +; CHECK-NEXT: [[CC:%.*]] = call i1 @getcc() +; CHECK-NEXT: br i1 [[CC]], label [[LOOP]], label [[END]] +; CHECK: end: +; CHECK-NEXT: [[PHI_END:%.*]] = phi <8 x float> [ [[SCALAR]], [[LOOP]] ], [ [[LOAD]], [[ENTRY]] ] +; CHECK-NEXT: [[SCALAR_END:%.*]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> [[PHI_END]], half 0xH312F, i32 1, i32 1) +; CHECK-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[PTR]], i32 4, i1 true, i32 1, i32 1, i32 0, <8 x float> [[SCALAR_END]]) +; CHECK-NEXT: ret void +; +entry: + %load = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr %ptr, i32 4, i1 false, i32 1, i32 0, i32 0) + %guard = call i1 @getcc() + br i1 %guard, label %loop, label %end + +loop: + %v.loop = phi <8 x float> [ %load, %entry ], [ %v.next, %loop ] + + %f = call <8 x float> @getmat1() + %pre = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %v.loop, i32 1, i32 1, i32 0, i32 1) + %muladd = call <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float> %f, <8 x float> %f, <8 x float> %pre, i1 true, i1 true, i1 false, i1 false, i32 1, i32 1) + %binop = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> %muladd, <8 x float> %muladd, i32 1, i32 1) + %scalar = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %binop, half 0xH310F, i32 1, i32 1) + %v.next = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %scalar, i32 1, i32 1, i32 1, i32 0) + + %cc = call i1 @getcc() + br i1 %cc, label %loop, label %end + +end: + %phi.end = phi <8 x float> [ %v.next, %loop ], [ %load, %entry ] + %scalar.end = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %phi.end, half 0xH312F, i32 1, i32 0) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %ptr, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %scalar.end) + ret void +} + +define void @convert_to_fact_inner_chain(ptr %ptr) { +; CHECK-LABEL: define void @convert_to_fact_inner_chain +; CHECK-SAME: (ptr [[PTR:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LOAD:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[PTR]], i32 4, i1 false, i32 1, i32 0, i32 0) +; CHECK-NEXT: [[GUARD:%.*]] = call i1 @getcc() +; CHECK-NEXT: br i1 [[GUARD]], label [[LOOP:%.*]], label [[END:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[V_LOOP:%.*]] = phi <8 x float> [ [[LOAD]], [[ENTRY:%.*]] ], [ [[SCALAR:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[F:%.*]] = call <8 x float> @getmat1() +; CHECK-NEXT: [[BINOP:%.*]] = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> [[V_LOOP]], <8 x float> [[V_LOOP]], i32 1, i32 0) +; CHECK-NEXT: [[SCALAR]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> [[BINOP]], half 0xH310F, i32 1, i32 0) +; CHECK-NEXT: [[CC:%.*]] = call i1 @getcc() +; CHECK-NEXT: br i1 [[CC]], label [[LOOP]], label [[END]] +; CHECK: end: +; CHECK-NEXT: [[PHI_END:%.*]] = phi <8 x float> [ [[SCALAR]], [[LOOP]] ], [ [[LOAD]], [[ENTRY]] ] +; CHECK-NEXT: [[SCALAR_END:%.*]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> [[PHI_END]], half 0xH312F, i32 1, i32 0) +; CHECK-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[PTR]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[SCALAR_END]]) +; CHECK-NEXT: ret void +; +entry: + %load = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr %ptr, i32 4, i1 false, i32 1, i32 1, i32 0) + %guard = call i1 @getcc() + br i1 %guard, label %loop, label %end + +loop: + %v.loop = phi <8 x float> [ %load, %entry ], [ %v.next, %loop ] + + %f = call <8 x float> @getmat1() + %pre = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %v.loop, i32 1, i32 1, i32 1, i32 0) + %binop = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> %pre, <8 x float> %pre, i32 1, i32 0) + %scalar = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %binop, half 0xH310F, i32 1, i32 0) + %v.next = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %scalar, i32 1, i32 1, i32 0, i32 1) + + %cc = call i1 @getcc() + br i1 %cc, label %loop, label %end + +end: + %phi.end = phi <8 x float> [ %v.next, %loop ], [ %load, %entry ] + %scalar.end = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %phi.end, half 0xH312F, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %ptr, i32 4, i1 true, i32 1, i32 1, i32 0, <8 x float> %scalar.end) + ret void +} + +define void @transpose_fact_inner_binop(ptr %ptr) { +; CHECK-LABEL: define void @transpose_fact_inner_binop +; CHECK-SAME: (ptr [[PTR:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LOAD:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[PTR]], i32 4, i1 true, i32 1, i32 0, i32 0) +; CHECK-NEXT: [[BINOP:%.*]] = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> [[LOAD]], <8 x float> [[LOAD]], i32 1, i32 0) +; CHECK-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[PTR]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[LOAD]]) +; CHECK-NEXT: ret void +; +entry: + %load = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr %ptr, i32 4, i1 false, i32 1, i32 0, i32 0) + %trans.pre = call <8 x float> @lgc.cooperative.matrix.transpose.v8f32.v8f32.i32.i32(<8 x float> %load, i32 1, i32 0) + %binop = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> %trans.pre, <8 x float> %trans.pre, i32 1, i32 0) + %trans.post = call <8 x float> @lgc.cooperative.matrix.transpose.v8f32.v8f32.i32.i32(<8 x float> %load, i32 1, i32 0) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %ptr, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %trans.post) + ret void +} + +define void @transpose_acc_inner_binop(ptr %ptr) { +; CHECK-LABEL: define void @transpose_acc_inner_binop +; CHECK-SAME: (ptr [[PTR:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LOAD:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[PTR]], i32 4, i1 true, i32 1, i32 1, i32 0) +; CHECK-NEXT: [[BINOP:%.*]] = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> [[LOAD]], <8 x float> [[LOAD]], i32 1, i32 1) +; CHECK-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[PTR]], i32 4, i1 true, i32 1, i32 1, i32 0, <8 x float> [[LOAD]]) +; CHECK-NEXT: ret void +; +entry: + %load = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr %ptr, i32 4, i1 false, i32 1, i32 1, i32 0) + %trans.pre = call <8 x float> @lgc.cooperative.matrix.transpose.v8f32.v8f32.i32.i32(<8 x float> %load, i32 1, i32 1) + %binop = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> %trans.pre, <8 x float> %trans.pre, i32 1, i32 1) + %trans.post = call <8 x float> @lgc.cooperative.matrix.transpose.v8f32.v8f32.i32.i32(<8 x float> %load, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %ptr, i32 4, i1 true, i32 1, i32 1, i32 0, <8 x float> %trans.post) + ret void +} + +define void @transpose_fact_inner_times_scalar(ptr %ptr) { +; CHECK-LABEL: define void @transpose_fact_inner_times_scalar +; CHECK-SAME: (ptr [[PTR:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LOAD:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[PTR]], i32 4, i1 true, i32 1, i32 0, i32 0) +; CHECK-NEXT: [[SCALAR:%.*]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> [[LOAD]], half 0xH310F, i32 1, i32 0) +; CHECK-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[PTR]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[SCALAR]]) +; CHECK-NEXT: ret void +; +entry: + %load = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr %ptr, i32 4, i1 false, i32 1, i32 0, i32 0) + %trans.pre = call <8 x float> @lgc.cooperative.matrix.transpose.v8f32.v8f32.i32.i32(<8 x float> %load, i32 1, i32 0) + %scalar = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %trans.pre, half 0xH310F, i32 1, i32 0) + %trans.post = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %scalar, i32 1, i32 1, i32 1, i32 0) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %ptr, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %trans.post) + ret void +} + +define void @transpose_acc_inner_times_scalar(ptr %ptr) { +; CHECK-LABEL: define void @transpose_acc_inner_times_scalar +; CHECK-SAME: (ptr [[PTR:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LOAD:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[PTR]], i32 4, i1 true, i32 1, i32 1, i32 0) +; CHECK-NEXT: [[SCALAR:%.*]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> [[LOAD]], half 0xH310F, i32 1, i32 1) +; CHECK-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[PTR]], i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> [[SCALAR]]) +; CHECK-NEXT: ret void +; +entry: + %load = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr %ptr, i32 4, i1 false, i32 1, i32 1, i32 0) + %trans.pre = call <8 x float> @lgc.cooperative.matrix.transpose.v8f32.v8f32.i32.i32(<8 x float> %load, i32 1, i32 1) + %scalar = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %trans.pre, half 0xH310F, i32 1, i32 1) + %trans.post = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %scalar, i32 1, i32 1, i32 1, i32 1) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %ptr, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %trans.post) + ret void +} + +define void @transpose_inner_chain(ptr %ptr) { +; CHECK-LABEL: define void @transpose_inner_chain +; CHECK-SAME: (ptr [[PTR:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LOAD:%.*]] = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr [[PTR]], i32 4, i1 true, i32 1, i32 0, i32 0) +; CHECK-NEXT: [[GUARD:%.*]] = call i1 @getcc() +; CHECK-NEXT: br i1 [[GUARD]], label [[LOOP:%.*]], label [[END:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[V_LOOP:%.*]] = phi <8 x float> [ [[LOAD]], [[ENTRY:%.*]] ], [ [[SCALAR:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[F:%.*]] = call <8 x float> @getmat1() +; CHECK-NEXT: [[BINOP:%.*]] = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> [[V_LOOP]], <8 x float> [[V_LOOP]], i32 1, i32 0) +; CHECK-NEXT: [[SCALAR]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> [[BINOP]], half 0xH310F, i32 1, i32 0) +; CHECK-NEXT: [[CC:%.*]] = call i1 @getcc() +; CHECK-NEXT: br i1 [[CC]], label [[LOOP]], label [[END]] +; CHECK: end: +; CHECK-NEXT: [[PHI_END:%.*]] = phi <8 x float> [ [[SCALAR]], [[LOOP]] ], [ [[LOAD]], [[ENTRY]] ] +; CHECK-NEXT: [[SCALAR_END:%.*]] = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> [[PHI_END]], half 0xH312F, i32 1, i32 0) +; CHECK-NEXT: call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr [[PTR]], i32 4, i1 false, i32 1, i32 0, i32 0, <8 x float> [[SCALAR_END]]) +; CHECK-NEXT: ret void +; +entry: + %load = call <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr %ptr, i32 4, i1 false, i32 1, i32 0, i32 0) + %guard = call i1 @getcc() + br i1 %guard, label %loop, label %end + +loop: + %v.loop = phi <8 x float> [ %load, %entry ], [ %v.next, %loop ] + + %f = call <8 x float> @getmat1() + %trans.pre = call <8 x float> @lgc.cooperative.matrix.transpose.v8f32.v8f32.i32.i32(<8 x float> %v.loop, i32 1, i32 0) + %binop = call <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32 1, <8 x float> %trans.pre, <8 x float> %trans.pre, i32 1, i32 0) + %scalar = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %binop, half 0xH310F, i32 1, i32 0) + %v.next = call <8 x float> @lgc.cooperative.matrix.transpose.v8f32.v8f32.i32.i32(<8 x float> %scalar, i32 1, i32 0) + + %cc = call i1 @getcc() + br i1 %cc, label %loop, label %end + +end: + %phi.end = phi <8 x float> [ %v.next, %loop ], [ %load, %entry ] + %scalar.end = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float> %phi.end, half 0xH312F, i32 1, i32 0) + call void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr %ptr, i32 4, i1 true, i32 1, i32 0, i32 0, <8 x float> %scalar.end) + ret void +} + declare i1 @getcc() declare <8 x float> @getmat1() declare <8 x float> @lgc.cooperative.matrix.load.v8f32.p0.i32.i1.i32.i32.i32(ptr, i32, i1, i32, i32, i32) declare <8 x float> @lgc.cooperative.matrix.transpose.v8f32.v8f32.i32.i32(<8 x float>, i32, i32) declare <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32, <8 x float>, i32, i32, i32, i32) -declare <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i32.i32(<8 x float>, <8 x float>, <8 x float>, i1, i1, i32, i32) +declare <8 x float> @lgc.cooperative.matrix.muladd.v8f32.v8f32.v8f32.v8f32.i1.i1.i1.i1.i32.i32(<8 x float>, <8 x float>, <8 x float>, i1, i1, i1, i1, i32, i32) +declare <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.f16.i32.i32(<8 x float>, half, i32, i32) +declare <8 x float> @lgc.cooperative.matrix.binop.v8f32.i32.v8f32.v8f32.i32.i32(i32, <8 x float>, <8 x float>, i32, i32) +declare void @lgc.cooperative.matrix.store.p0.i32.i1.i32.i32.i32.v8f32(ptr, i32, i1, i32, i32, i32, <8 x float>) diff --git a/lgc/test/Transforms/LowerCooperativeMatrix/convert.lgc b/lgc/test/Transforms/LowerCooperativeMatrix/convert.lgc index dcbf7ddee2..32bb1988f8 100644 --- a/lgc/test/Transforms/LowerCooperativeMatrix/convert.lgc +++ b/lgc/test/Transforms/LowerCooperativeMatrix/convert.lgc @@ -1,128 +1,54 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool lgc ; RUN: lgc -o - -passes=lgc-lower-cooperative-matrix %s | FileCheck --check-prefixes=CHECK %s -define <8 x float> @test_relayout_simple(<8 x float> %ab) { -; CHECK-LABEL: @test_relayout_simple( +define <8 x float> @convert_f16_to_accumulator(<8 x float> %fact) { +; CHECK-LABEL: @convert_f16_to_accumulator( ; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) ; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[TMP1]]) ; CHECK-NEXT: [[TMP3:%.*]] = udiv i32 [[TMP2]], 16 ; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[TMP3]], 1 ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[TMP4]], 0 -; CHECK-NEXT: [[TMP6:%.*]] = bitcast <8 x float> [[AB:%.*]] to <8 x i32> -; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x float> [[AB]], i64 0 -; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x float> [[AB]], i64 1 +; CHECK-NEXT: [[TMP6:%.*]] = bitcast <8 x float> [[FACT:%.*]] to <8 x i32> +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x float> [[FACT]], i64 0 +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x float> [[FACT]], i64 1 ; CHECK-NEXT: [[TMP9:%.*]] = and i32 [[TMP3]], 2 ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP9]], 0 ; CHECK-NEXT: [[TMP11:%.*]] = select i1 [[TMP10]], float [[TMP7]], float [[TMP8]] -; CHECK-NEXT: [[B1:%.*]] = insertelement <4 x float> poison, float [[TMP11]], i64 0 -; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x float> [[AB]], i64 2 -; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x float> [[AB]], i64 3 +; CHECK-NEXT: [[ACCUM1:%.*]] = insertelement <4 x float> poison, float [[TMP11]], i64 0 +; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x float> [[FACT]], i64 2 +; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x float> [[FACT]], i64 3 ; CHECK-NEXT: [[TMP14:%.*]] = and i32 [[TMP3]], 2 ; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i32 [[TMP14]], 0 ; CHECK-NEXT: [[TMP16:%.*]] = select i1 [[TMP15]], float [[TMP12]], float [[TMP13]] -; CHECK-NEXT: [[B2:%.*]] = insertelement <4 x float> [[B1]], float [[TMP16]], i64 1 -; CHECK-NEXT: [[TMP17:%.*]] = extractelement <8 x float> [[AB]], i64 4 -; CHECK-NEXT: [[TMP18:%.*]] = extractelement <8 x float> [[AB]], i64 5 +; CHECK-NEXT: [[ACCUM2:%.*]] = insertelement <4 x float> [[ACCUM1]], float [[TMP16]], i64 1 +; CHECK-NEXT: [[TMP17:%.*]] = extractelement <8 x float> [[FACT]], i64 4 +; CHECK-NEXT: [[TMP18:%.*]] = extractelement <8 x float> [[FACT]], i64 5 ; CHECK-NEXT: [[TMP19:%.*]] = and i32 [[TMP3]], 2 ; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i32 [[TMP19]], 0 ; CHECK-NEXT: [[TMP21:%.*]] = select i1 [[TMP20]], float [[TMP17]], float [[TMP18]] -; CHECK-NEXT: [[B3:%.*]] = insertelement <4 x float> [[B2]], float [[TMP21]], i64 2 -; CHECK-NEXT: [[TMP22:%.*]] = extractelement <8 x float> [[AB]], i64 6 -; CHECK-NEXT: [[TMP23:%.*]] = extractelement <8 x float> [[AB]], i64 7 +; CHECK-NEXT: [[ACCUM3:%.*]] = insertelement <4 x float> [[ACCUM2]], float [[TMP21]], i64 2 +; CHECK-NEXT: [[TMP22:%.*]] = extractelement <8 x float> [[FACT]], i64 6 +; CHECK-NEXT: [[TMP23:%.*]] = extractelement <8 x float> [[FACT]], i64 7 ; CHECK-NEXT: [[TMP24:%.*]] = and i32 [[TMP3]], 2 ; CHECK-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP24]], 0 ; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP25]], float [[TMP22]], float [[TMP23]] -; CHECK-NEXT: [[B4:%.*]] = insertelement <4 x float> [[B3]], float [[TMP26]], i64 3 -; CHECK-NEXT: [[TMP27:%.*]] = bitcast <4 x float> [[B4]] to <4 x i32> +; CHECK-NEXT: [[ACCUM4:%.*]] = insertelement <4 x float> [[ACCUM3]], float [[TMP26]], i64 3 +; CHECK-NEXT: [[TMP27:%.*]] = bitcast <4 x float> [[ACCUM4]] to <4 x i32> ; CHECK-NEXT: [[TMP28:%.*]] = select i1 [[TMP5]], <4 x i32> zeroinitializer, <4 x i32> -; CHECK-NEXT: [[B5:%.*]] = lshr <4 x i32> [[TMP27]], [[TMP28]] -; CHECK-NEXT: [[B6:%.*]] = bitcast <4 x i32> [[B5]] to <4 x float> -; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <4 x float> [[B6]], <4 x float> poison, <8 x i32> -; CHECK-NEXT: [[TMP30:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) -; CHECK-NEXT: [[TMP31:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[TMP30]]) -; CHECK-NEXT: [[TMP32:%.*]] = bitcast <8 x float> [[TMP29]] to <8 x i32> -; CHECK-NEXT: [[TMP33:%.*]] = udiv i32 [[TMP31]], 16 -; CHECK-NEXT: [[TMP34:%.*]] = and i32 [[TMP33]], 1 -; CHECK-NEXT: [[TMP35:%.*]] = icmp eq i32 [[TMP34]], 0 -; CHECK-NEXT: [[TMP36:%.*]] = shufflevector <8 x i32> [[TMP32]], <8 x i32> poison, <8 x i32> -; CHECK-NEXT: [[TMP37:%.*]] = extractelement <8 x i32> [[TMP36]], i64 0 -; CHECK-NEXT: [[TMP38:%.*]] = extractelement <8 x i32> [[TMP36]], i64 0 -; CHECK-NEXT: [[TMP39:%.*]] = call i32 @llvm.amdgcn.permlanex16(i32 [[TMP37]], i32 [[TMP38]], i32 1985229328, i32 -19088744, i1 false, i1 false) -; CHECK-NEXT: [[TMP40:%.*]] = extractelement <8 x i32> [[TMP36]], i64 1 -; CHECK-NEXT: [[TMP41:%.*]] = extractelement <8 x i32> [[TMP36]], i64 1 -; CHECK-NEXT: [[TMP42:%.*]] = call i32 @llvm.amdgcn.permlanex16(i32 [[TMP40]], i32 [[TMP41]], i32 1985229328, i32 -19088744, i1 false, i1 false) -; CHECK-NEXT: [[TMP43:%.*]] = extractelement <8 x i32> [[TMP36]], i64 2 -; CHECK-NEXT: [[TMP44:%.*]] = extractelement <8 x i32> [[TMP36]], i64 2 -; CHECK-NEXT: [[TMP45:%.*]] = call i32 @llvm.amdgcn.permlanex16(i32 [[TMP43]], i32 [[TMP44]], i32 1985229328, i32 -19088744, i1 false, i1 false) -; CHECK-NEXT: [[TMP46:%.*]] = extractelement <8 x i32> [[TMP36]], i64 3 -; CHECK-NEXT: [[TMP47:%.*]] = extractelement <8 x i32> [[TMP36]], i64 3 -; CHECK-NEXT: [[TMP48:%.*]] = call i32 @llvm.amdgcn.permlanex16(i32 [[TMP46]], i32 [[TMP47]], i32 1985229328, i32 -19088744, i1 false, i1 false) -; CHECK-NEXT: [[TMP49:%.*]] = extractelement <8 x i32> [[TMP36]], i64 4 -; CHECK-NEXT: [[TMP50:%.*]] = extractelement <8 x i32> [[TMP36]], i64 4 -; CHECK-NEXT: [[TMP51:%.*]] = call i32 @llvm.amdgcn.permlanex16(i32 [[TMP49]], i32 [[TMP50]], i32 1985229328, i32 -19088744, i1 false, i1 false) -; CHECK-NEXT: [[TMP52:%.*]] = extractelement <8 x i32> [[TMP36]], i64 5 -; CHECK-NEXT: [[TMP53:%.*]] = extractelement <8 x i32> [[TMP36]], i64 5 -; CHECK-NEXT: [[TMP54:%.*]] = call i32 @llvm.amdgcn.permlanex16(i32 [[TMP52]], i32 [[TMP53]], i32 1985229328, i32 -19088744, i1 false, i1 false) -; CHECK-NEXT: [[TMP55:%.*]] = extractelement <8 x i32> [[TMP36]], i64 6 -; CHECK-NEXT: [[TMP56:%.*]] = extractelement <8 x i32> [[TMP36]], i64 6 -; CHECK-NEXT: [[TMP57:%.*]] = call i32 @llvm.amdgcn.permlanex16(i32 [[TMP55]], i32 [[TMP56]], i32 1985229328, i32 -19088744, i1 false, i1 false) -; CHECK-NEXT: [[TMP58:%.*]] = extractelement <8 x i32> [[TMP36]], i64 7 -; CHECK-NEXT: [[TMP59:%.*]] = extractelement <8 x i32> [[TMP36]], i64 7 -; CHECK-NEXT: [[TMP60:%.*]] = call i32 @llvm.amdgcn.permlanex16(i32 [[TMP58]], i32 [[TMP59]], i32 1985229328, i32 -19088744, i1 false, i1 false) -; CHECK-NEXT: [[TMP61:%.*]] = insertelement <8 x i32> poison, i32 [[TMP39]], i64 0 -; CHECK-NEXT: [[TMP62:%.*]] = insertelement <8 x i32> [[TMP61]], i32 [[TMP42]], i64 1 -; CHECK-NEXT: [[TMP63:%.*]] = insertelement <8 x i32> [[TMP62]], i32 [[TMP45]], i64 2 -; CHECK-NEXT: [[TMP64:%.*]] = insertelement <8 x i32> [[TMP63]], i32 [[TMP48]], i64 3 -; CHECK-NEXT: [[TMP65:%.*]] = insertelement <8 x i32> [[TMP64]], i32 [[TMP51]], i64 4 -; CHECK-NEXT: [[TMP66:%.*]] = insertelement <8 x i32> [[TMP65]], i32 [[TMP54]], i64 5 -; CHECK-NEXT: [[TMP67:%.*]] = insertelement <8 x i32> [[TMP66]], i32 [[TMP57]], i64 6 -; CHECK-NEXT: [[TMP68:%.*]] = insertelement <8 x i32> [[TMP67]], i32 [[TMP60]], i64 7 -; CHECK-NEXT: [[TMP69:%.*]] = select i1 [[TMP35]], <8 x i32> [[TMP36]], <8 x i32> [[TMP68]] -; CHECK-NEXT: [[TMP70:%.*]] = select i1 [[TMP35]], <8 x i32> [[TMP68]], <8 x i32> [[TMP36]] -; CHECK-NEXT: [[TMP71:%.*]] = and <8 x i32> [[TMP69]], -; CHECK-NEXT: [[TMP72:%.*]] = shl <8 x i32> [[TMP70]], -; CHECK-NEXT: [[TMP73:%.*]] = or <8 x i32> [[TMP71]], [[TMP72]] -; CHECK-NEXT: [[TMP74:%.*]] = extractelement <8 x i32> [[TMP73]], i64 0 -; CHECK-NEXT: [[TMP75:%.*]] = call i32 @llvm.amdgcn.permlane64(i32 [[TMP74]]) -; CHECK-NEXT: [[TMP76:%.*]] = extractelement <8 x i32> [[TMP73]], i64 1 -; CHECK-NEXT: [[TMP77:%.*]] = call i32 @llvm.amdgcn.permlane64(i32 [[TMP76]]) -; CHECK-NEXT: [[TMP78:%.*]] = extractelement <8 x i32> [[TMP73]], i64 2 -; CHECK-NEXT: [[TMP79:%.*]] = call i32 @llvm.amdgcn.permlane64(i32 [[TMP78]]) -; CHECK-NEXT: [[TMP80:%.*]] = extractelement <8 x i32> [[TMP73]], i64 3 -; CHECK-NEXT: [[TMP81:%.*]] = call i32 @llvm.amdgcn.permlane64(i32 [[TMP80]]) -; CHECK-NEXT: [[TMP82:%.*]] = extractelement <8 x i32> [[TMP73]], i64 4 -; CHECK-NEXT: [[TMP83:%.*]] = call i32 @llvm.amdgcn.permlane64(i32 [[TMP82]]) -; CHECK-NEXT: [[TMP84:%.*]] = extractelement <8 x i32> [[TMP73]], i64 5 -; CHECK-NEXT: [[TMP85:%.*]] = call i32 @llvm.amdgcn.permlane64(i32 [[TMP84]]) -; CHECK-NEXT: [[TMP86:%.*]] = extractelement <8 x i32> [[TMP73]], i64 6 -; CHECK-NEXT: [[TMP87:%.*]] = call i32 @llvm.amdgcn.permlane64(i32 [[TMP86]]) -; CHECK-NEXT: [[TMP88:%.*]] = extractelement <8 x i32> [[TMP73]], i64 7 -; CHECK-NEXT: [[TMP89:%.*]] = call i32 @llvm.amdgcn.permlane64(i32 [[TMP88]]) -; CHECK-NEXT: [[TMP90:%.*]] = insertelement <8 x i32> poison, i32 [[TMP75]], i64 0 -; CHECK-NEXT: [[TMP91:%.*]] = insertelement <8 x i32> [[TMP90]], i32 [[TMP77]], i64 1 -; CHECK-NEXT: [[TMP92:%.*]] = insertelement <8 x i32> [[TMP91]], i32 [[TMP79]], i64 2 -; CHECK-NEXT: [[TMP93:%.*]] = insertelement <8 x i32> [[TMP92]], i32 [[TMP81]], i64 3 -; CHECK-NEXT: [[TMP94:%.*]] = insertelement <8 x i32> [[TMP93]], i32 [[TMP83]], i64 4 -; CHECK-NEXT: [[TMP95:%.*]] = insertelement <8 x i32> [[TMP94]], i32 [[TMP85]], i64 5 -; CHECK-NEXT: [[TMP96:%.*]] = insertelement <8 x i32> [[TMP95]], i32 [[TMP87]], i64 6 -; CHECK-NEXT: [[TMP97:%.*]] = insertelement <8 x i32> [[TMP96]], i32 [[TMP89]], i64 7 -; CHECK-NEXT: [[TMP98:%.*]] = icmp ult i32 [[TMP31]], 32 -; CHECK-NEXT: [[TMP99:%.*]] = select i1 [[TMP98]], <8 x i32> [[TMP73]], <8 x i32> [[TMP97]] -; CHECK-NEXT: [[TMP100:%.*]] = select i1 [[TMP98]], <8 x i32> [[TMP97]], <8 x i32> [[TMP73]] -; CHECK-NEXT: [[C7:%.*]] = shufflevector <8 x i32> [[TMP99]], <8 x i32> [[TMP100]], <8 x i32> -; CHECK-NEXT: [[TMP101:%.*]] = bitcast <8 x i32> [[C7]] to <8 x float> -; CHECK-NEXT: ret <8 x float> [[TMP101]] +; CHECK-NEXT: [[ACCUM5:%.*]] = lshr <4 x i32> [[TMP27]], [[TMP28]] +; CHECK-NEXT: [[ACCUM6:%.*]] = bitcast <4 x i32> [[ACCUM5]] to <4 x float> +; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <4 x float> [[ACCUM6]], <4 x float> poison, <8 x i32> +; CHECK-NEXT: ret <8 x float> [[TMP29]] ; - %b = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %ab, i32 1, i32 1, i32 0, i32 1) - %c = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %b, i32 1, i32 1, i32 1, i32 0) - ret <8 x float> %c + %accum = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %fact, i32 1, i32 1, i32 0, i32 1) + ret <8 x float> %accum } -define <8 x float> @test_relayout_simple_reverse(<8 x float> %cd) { -; CHECK-LABEL: @test_relayout_simple_reverse( +define <8 x float> @convert_f16_to_factor(<8 x float> %accum) { +; CHECK-LABEL: @convert_f16_to_factor( ; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) ; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[TMP1]]) -; CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x float> [[CD:%.*]] to <8 x i32> +; CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x float> [[ACCUM:%.*]] to <8 x i32> ; CHECK-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP2]], 16 ; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[TMP4]], 1 ; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0 @@ -191,48 +117,12 @@ define <8 x float> @test_relayout_simple_reverse(<8 x float> %cd) { ; CHECK-NEXT: [[TMP69:%.*]] = icmp ult i32 [[TMP2]], 32 ; CHECK-NEXT: [[TMP70:%.*]] = select i1 [[TMP69]], <8 x i32> [[TMP44]], <8 x i32> [[TMP68]] ; CHECK-NEXT: [[TMP71:%.*]] = select i1 [[TMP69]], <8 x i32> [[TMP68]], <8 x i32> [[TMP44]] -; CHECK-NEXT: [[B1:%.*]] = shufflevector <8 x i32> [[TMP70]], <8 x i32> [[TMP71]], <8 x i32> -; CHECK-NEXT: [[TMP72:%.*]] = bitcast <8 x i32> [[B1]] to <8 x float> -; CHECK-NEXT: [[TMP73:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) -; CHECK-NEXT: [[TMP74:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[TMP73]]) -; CHECK-NEXT: [[TMP75:%.*]] = udiv i32 [[TMP74]], 16 -; CHECK-NEXT: [[TMP76:%.*]] = and i32 [[TMP75]], 1 -; CHECK-NEXT: [[TMP77:%.*]] = icmp eq i32 [[TMP76]], 0 -; CHECK-NEXT: [[TMP78:%.*]] = bitcast <8 x float> [[TMP72]] to <8 x i32> -; CHECK-NEXT: [[TMP79:%.*]] = extractelement <8 x float> [[TMP72]], i64 0 -; CHECK-NEXT: [[TMP80:%.*]] = extractelement <8 x float> [[TMP72]], i64 1 -; CHECK-NEXT: [[TMP81:%.*]] = and i32 [[TMP75]], 2 -; CHECK-NEXT: [[TMP82:%.*]] = icmp eq i32 [[TMP81]], 0 -; CHECK-NEXT: [[TMP83:%.*]] = select i1 [[TMP82]], float [[TMP79]], float [[TMP80]] -; CHECK-NEXT: [[C2:%.*]] = insertelement <4 x float> poison, float [[TMP83]], i64 0 -; CHECK-NEXT: [[TMP84:%.*]] = extractelement <8 x float> [[TMP72]], i64 2 -; CHECK-NEXT: [[TMP85:%.*]] = extractelement <8 x float> [[TMP72]], i64 3 -; CHECK-NEXT: [[TMP86:%.*]] = and i32 [[TMP75]], 2 -; CHECK-NEXT: [[TMP87:%.*]] = icmp eq i32 [[TMP86]], 0 -; CHECK-NEXT: [[TMP88:%.*]] = select i1 [[TMP87]], float [[TMP84]], float [[TMP85]] -; CHECK-NEXT: [[C3:%.*]] = insertelement <4 x float> [[C2]], float [[TMP88]], i64 1 -; CHECK-NEXT: [[TMP89:%.*]] = extractelement <8 x float> [[TMP72]], i64 4 -; CHECK-NEXT: [[TMP90:%.*]] = extractelement <8 x float> [[TMP72]], i64 5 -; CHECK-NEXT: [[TMP91:%.*]] = and i32 [[TMP75]], 2 -; CHECK-NEXT: [[TMP92:%.*]] = icmp eq i32 [[TMP91]], 0 -; CHECK-NEXT: [[TMP93:%.*]] = select i1 [[TMP92]], float [[TMP89]], float [[TMP90]] -; CHECK-NEXT: [[C4:%.*]] = insertelement <4 x float> [[C3]], float [[TMP93]], i64 2 -; CHECK-NEXT: [[TMP94:%.*]] = extractelement <8 x float> [[TMP72]], i64 6 -; CHECK-NEXT: [[TMP95:%.*]] = extractelement <8 x float> [[TMP72]], i64 7 -; CHECK-NEXT: [[TMP96:%.*]] = and i32 [[TMP75]], 2 -; CHECK-NEXT: [[TMP97:%.*]] = icmp eq i32 [[TMP96]], 0 -; CHECK-NEXT: [[TMP98:%.*]] = select i1 [[TMP97]], float [[TMP94]], float [[TMP95]] -; CHECK-NEXT: [[C5:%.*]] = insertelement <4 x float> [[C4]], float [[TMP98]], i64 3 -; CHECK-NEXT: [[TMP99:%.*]] = bitcast <4 x float> [[C5]] to <4 x i32> -; CHECK-NEXT: [[TMP100:%.*]] = select i1 [[TMP77]], <4 x i32> zeroinitializer, <4 x i32> -; CHECK-NEXT: [[C6:%.*]] = lshr <4 x i32> [[TMP99]], [[TMP100]] -; CHECK-NEXT: [[C7:%.*]] = bitcast <4 x i32> [[C6]] to <4 x float> -; CHECK-NEXT: [[TMP101:%.*]] = shufflevector <4 x float> [[C7]], <4 x float> poison, <8 x i32> -; CHECK-NEXT: ret <8 x float> [[TMP101]] +; CHECK-NEXT: [[FACT1:%.*]] = shufflevector <8 x i32> [[TMP70]], <8 x i32> [[TMP71]], <8 x i32> +; CHECK-NEXT: [[TMP72:%.*]] = bitcast <8 x i32> [[FACT1]] to <8 x float> +; CHECK-NEXT: ret <8 x float> [[TMP72]] ; - %b = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %cd, i32 1, i32 1, i32 1, i32 0) - %c = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %b, i32 1, i32 1, i32 0, i32 1) - ret <8 x float> %c + %fact = call <8 x float> @lgc.cooperative.matrix.convert.v8f32.i32.v8f32.i32.i32.i32.i32(i32 0, <8 x float> %accum, i32 1, i32 1, i32 1, i32 0) + ret <8 x float> %fact } declare i1 @getcc() diff --git a/lgc/test/Transforms/LowerCooperativeMatrix/packed-accumulators-wave64.lgc b/lgc/test/Transforms/LowerCooperativeMatrix/packed-accumulators-wave64.lgc new file mode 100644 index 0000000000..3639dd82b6 --- /dev/null +++ b/lgc/test/Transforms/LowerCooperativeMatrix/packed-accumulators-wave64.lgc @@ -0,0 +1,79 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --tool lgc --version 2 +; RUN: lgc --mcpu=gfx1100 -o - -passes=lgc-lower-cooperative-matrix %s | FileCheck --check-prefixes=CHECK %s + +define <8 x float> @test_pack_f16(<8 x float> %a, <8 x float> %b) !spirv.ExecutionModel !8 !lgc.shaderstage !9 { +; CHECK-LABEL: define <8 x float> @test_pack_f16 +; CHECK-SAME: (<8 x float> [[A:%.*]], <8 x float> [[B:%.*]]) !spirv.ExecutionModel !5 !lgc.shaderstage !6 { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x float> [[A]] to <16 x half> +; CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x float> [[B]] to <16 x half> +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> [[TMP1]], <16 x i32> +; CHECK-NEXT: [[TMP3:%.*]] = bitcast <16 x half> [[TMP2]] to <8 x float> +; CHECK-NEXT: ret <8 x float> [[TMP3]] +; +entry: + %r = call <8 x float > @lgc.cooperative.matrix.pack.v8f32.v8f32(<8 x float> %a, <8 x float> %b) + ret <8 x float> %r +} + +define <8 x float> @test_unpack_lo(<8 x float> %packed) !spirv.ExecutionModel !8 !lgc.shaderstage !9 { +; CHECK-LABEL: define <8 x float> @test_unpack_lo +; CHECK-SAME: (<8 x float> [[PACKED:%.*]]) !spirv.ExecutionModel !5 !lgc.shaderstage !6 { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x float> [[PACKED]] to <16 x half> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <16 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = bitcast <16 x half> [[TMP1]] to <8 x float> +; CHECK-NEXT: ret <8 x float> [[TMP2]] +; +entry: + %r = call <8 x float >@lgc.cooperative.matrix.unpack.v8f32.i1(<8 x float> %packed, i1 false) + ret <8 x float> %r +} + +define <8 x float> @test_unpack_hi(<8 x float> %packed) !spirv.ExecutionModel !8 !lgc.shaderstage !9 { +; CHECK-LABEL: define <8 x float> @test_unpack_hi +; CHECK-SAME: (<8 x float> [[PACKED:%.*]]) !spirv.ExecutionModel !5 !lgc.shaderstage !6 { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x float> [[PACKED]] to <16 x half> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <16 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = bitcast <16 x half> [[TMP1]] to <8 x float> +; CHECK-NEXT: ret <8 x float> [[TMP2]] +; +entry: + %r = call <8 x float >@lgc.cooperative.matrix.unpack.v8f32.i1(<8 x float> %packed, i1 true) + ret <8 x float> %r +} + +define <8 x float> @test_packed_times_scalar(<8 x float> %packed, <2 x half> %scalar) !spirv.ExecutionModel !8 !lgc.shaderstage !9 { +; CHECK-LABEL: define <8 x float> @test_packed_times_scalar +; CHECK-SAME: (<8 x float> [[PACKED:%.*]], <2 x half> [[SCALAR:%.*]]) !spirv.ExecutionModel !5 !lgc.shaderstage !6 { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x float> [[PACKED]] to <16 x half> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x half> [[TMP0]], <16 x half> poison, <8 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x half> [[SCALAR]], <2 x half> poison, <8 x i32> +; CHECK-NEXT: [[TMP3:%.*]] = fmul <8 x half> [[TMP1]], [[TMP2]] +; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x half> [[TMP3]], <8 x half> poison, <16 x i32> +; CHECK-NEXT: [[TMP5:%.*]] = bitcast <16 x half> [[TMP4]] to <8 x float> +; CHECK-NEXT: ret <8 x float> [[TMP5]] +; +entry: + %r = call <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.v2f16.i32.i32(<8 x float> %packed, <2 x half> %scalar, i32 6, i32 1) + ret <8 x float> %r +} + +declare <8 x float> @lgc.cooperative.matrix.pack.v8f32.v8f32(<8 x float>, <8 x float>) +declare <8 x float> @lgc.cooperative.matrix.unpack.v8f32.i1(<8 x float>, i1) +declare <8 x float> @lgc.cooperative.matrix.times.scalar.v8f32.v8f32.v2f16.i32.i32(<8 x float>, <2 x half>, i32, i32) + +!llpc.compute.mode = !{!0} +!lgc.client = !{!1} +!lgc.options = !{!2} +!lgc.options.CS = !{!3} + +!0 = !{i32 128, i32 2, i32 1} +!1 = !{!"Vulkan"} +!2 = !{i32 -2108299168, i32 -1199997545, i32 1667044824, i32 -422575072, i32 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 0, i32 0, i32 -1} +!3 = !{i32 219437737, i32 -1317595285, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 64, i32 64, i32 0, i32 0, i32 3, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 20, i32 1800} +!4 = !{i32 32, i32 1, i32 1} +!8 = !{i32 5} +!9 = !{i32 7} diff --git a/lgc/test/lit.cfg.py b/lgc/test/lit.cfg.py index 36a44b9e14..ef5bf1faa5 100644 --- a/lgc/test/lit.cfg.py +++ b/lgc/test/lit.cfg.py @@ -39,4 +39,3 @@ llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True) llvm_config.use_default_substitutions() - diff --git a/lgc/util/ModuleBunch.cpp b/lgc/util/ModuleBunch.cpp index d1eca1c14d..dd2a870202 100644 --- a/lgc/util/ModuleBunch.cpp +++ b/lgc/util/ModuleBunch.cpp @@ -288,7 +288,7 @@ std::optional> MbPassBuilder::parsePip } if (PipelineStack.size() > 1) - // Unbalanced paretheses. + // Unbalanced parentheses. return std::nullopt; assert(PipelineStack.back() == &ResultPipeline && "Wrong pipeline at the bottom of the stack!"); diff --git a/llpc/context/llpcCompiler.cpp b/llpc/context/llpcCompiler.cpp index d21c447006..3fda5ead00 100644 --- a/llpc/context/llpcCompiler.cpp +++ b/llpc/context/llpcCompiler.cpp @@ -520,7 +520,7 @@ void Compiler::Destroy() { // // @param codeBuffer : Spirv binary // @param imageSymbolInfo : Image symbol infos -static void mergeSpirvLocationAndBinding(llvm::MutableArrayRef codeBuffer, +static void mergeSpirvLocationAndBinding(MutableArrayRef codeBuffer, std::vector &imageSymbolInfo) { constexpr unsigned wordSize = sizeof(unsigned); @@ -558,6 +558,61 @@ static void mergeSpirvLocationAndBinding(llvm::MutableArrayRef codeBuf } } +// ===================================================================================================================== +// Recalculate resource binding for separate shader object +// +// @param codeBuffer : Spirv binary +// @param resourceBindingOffset : resource binding offset +// @param symbolInfo : resource symbol infos +static void recalcResourceBinding(MutableArrayRef codeBuffer, unsigned resourceBindingOffset, + std::vector &uniformBufferInfo, + std::vector &storageBufferInfo, + std::vector &textureSymbolInfo, + std::vector &imageSymbolInfo, + std::vector &atomicCounterSymbolInfo) { + constexpr unsigned wordSize = sizeof(unsigned); + + unsigned *code = codeBuffer.data(); + unsigned *end = code + codeBuffer.size(); + unsigned *codePos = code + sizeof(SpirvHeader) / wordSize; + + auto updateResourceBinding = [resourceBindingOffset](uint32_t varId, uint32_t binding, + std::vector &symbolInfo) { + for (auto it = symbolInfo.begin(); it != symbolInfo.end(); ++it) { + if (it->spvId == varId && it->binding == binding) { + it->binding = resourceBindingOffset << 16 | binding; + } + } + }; + + while (codePos < end) { + unsigned opCode = (codePos[0] & OpCodeMask); + unsigned wordCount = (codePos[0] >> WordCountShift); + + switch (opCode) { + case OpDecorate: { + auto decoration = static_cast(codePos[2]); + + if (decoration == DecorationBinding) { + uint32_t varId = codePos[1]; + uint32_t binding = codePos[3]; + + updateResourceBinding(varId, binding, uniformBufferInfo); + updateResourceBinding(varId, binding, storageBufferInfo); + updateResourceBinding(varId, binding, textureSymbolInfo); + updateResourceBinding(varId, binding, imageSymbolInfo); + updateResourceBinding(varId, binding, atomicCounterSymbolInfo); + codePos[3] = resourceBindingOffset << 16 | binding; + } + } break; + default: + break; + } + + codePos += wordCount; + } +} + // ===================================================================================================================== // Builds shader module from the specified info. // @@ -612,7 +667,13 @@ Result Compiler::BuildShaderModule(const ShaderModuleBuildInfo *shaderInfo, Shad allocSize += atomicCounterSymbolInfo.size() * sizeof(ResourceNodeData); allocSize += defaultUniformSymbolInfo.size() * sizeof(ResourceNodeData); - if (imageSymbolInfo.size() && shaderInfo->options.mergeLocationAndBinding) + // Solve binding conflictions for separate shader object + if (shaderInfo->options.resourceBindingOffset > 0) { + recalcResourceBinding(codeBuffer, shaderInfo->options.resourceBindingOffset, uniformBufferInfo, storageBufferInfo, + textureSymbolInfo, imageSymbolInfo, atomicCounterSymbolInfo); + } else if (imageSymbolInfo.size() && shaderInfo->options.mergeLocationAndBinding) + // Merge location and binding if image binding doesn't exist, the issue only exist on spirv binary cases, + // separate shader object doesn't support spirv binary, so it doesn't have such issue mergeSpirvLocationAndBinding(codeBuffer, imageSymbolInfo); } @@ -2856,6 +2917,9 @@ Result Compiler::buildRayTracingPipelineInternal(RayTracingContext &rtContext, bool needGpurtShaderLibrary = false; std::vector> modules(shaderInfo.size()); mainContext->setBuilder(builderContext->createBuilder(&*pipeline)); + const bool needGpurtForContinuations = (pipelineInfo->mode == Vkgc::LlpcRaytracingMode::Continuations); + if (needGpurtForContinuations) + needGpurtShaderLibrary = true; // Create empty modules and set target machine in each. for (unsigned shaderIndex = 0; shaderIndex < shaderInfo.size(); ++shaderIndex) { @@ -2930,7 +2994,7 @@ Result Compiler::buildRayTracingPipelineInternal(RayTracingContext &rtContext, const ShaderModuleData *moduleData = reinterpret_cast(shaderInfoEntry->pModuleData); auto shaderModule = std::move(modules[shaderIndex]); - if (moduleData->usage.enableRayQuery) { + if (moduleData->usage.enableRayQuery || needGpurtForContinuations) { Linker linker(*shaderModule); if (linker.linkInModule(CloneModule(*gpurtShaderLibrary))) return Result::ErrorInvalidShader; @@ -2941,20 +3005,28 @@ Result Compiler::buildRayTracingPipelineInternal(RayTracingContext &rtContext, moduleUsesRayQuery.push_back(moduleData->usage.enableRayQuery); } + // TODO: For continuations, we only need to compile the GpuRt module separately if there are TraceRay usages + // to compile the Traversal shader. For callable shaders, it is not required. if (gpurtShaderLibrary) { StringRef traceRayFuncName = mainContext->getPipelineContext()->getRayTracingFunctionName(Vkgc::RT_ENTRY_TRACE_RAY); StringRef fetchTrianglePosFunc = mainContext->getPipelineContext()->getRayTracingFunctionName( Vkgc::RT_ENTRY_FETCH_HIT_TRIANGLE_FROM_NODE_POINTER); - // NOTE: The GPURT shader library generated by DXC will contain some global constant value (0, 1, 2, etc.) shared - // across different functions. SpirvLowerGlobal pass cannot handle such case, so we drop all unneeded functions. + // Prepare GpuRt module to be compiled separately for (auto funcIt = gpurtShaderLibrary->begin(), funcEnd = gpurtShaderLibrary->end(); funcIt != funcEnd;) { Function *func = &*funcIt++; - if (func->getLinkage() == GlobalValue::ExternalLinkage && !func->empty()) { - if (!func->getName().startswith(traceRayFuncName) && !func->getName().startswith(fetchTrianglePosFunc)) { - func->dropAllReferences(); - func->eraseFromParent(); - } + if (func->getName().startswith(traceRayFuncName)) { + // We assigned GpuRt functions weak linkage prior to linking into app modules to not confuse the entry + // point determination mechanism. Undo that on TraceRay to make it the entry of the module. + func->setLinkage(GlobalValue::ExternalLinkage); + lgc::rt::setLgcRtShaderStage(func, lgc::rt::RayTracingShaderStage::Traversal); + } else if (func->getLinkage() == GlobalValue::WeakAnyLinkage && + !func->getName().startswith(fetchTrianglePosFunc) && !func->empty()) { + // Preserve fetchTrianglePosFunc because we need to inline it into Traversal later on. + // Remove other function definitions both for compile speed, and to work around an + // issue with private globals used in multiple functions in GpuRt which confuses SpirvLowerGlobal. + func->dropAllReferences(); + func->eraseFromParent(); } } diff --git a/llpc/context/llpcGraphicsContext.cpp b/llpc/context/llpcGraphicsContext.cpp index 47c88ae1b3..e9aa0de691 100644 --- a/llpc/context/llpcGraphicsContext.cpp +++ b/llpc/context/llpcGraphicsContext.cpp @@ -60,7 +60,7 @@ static cl::opt DisableColorExportShader("disable-color-export-shader", cl: GraphicsContext::GraphicsContext(GfxIpVersion gfxIp, const GraphicsPipelineBuildInfo *pipelineInfo, MetroHash::Hash *pipelineHash, MetroHash::Hash *cacheHash) : PipelineContext(gfxIp, pipelineHash, cacheHash), m_pipelineInfo(pipelineInfo), m_stageMask(0), - m_preRasterHasGs(false), m_useDualSourceBlend(false), m_activeStageCount(0) { + m_preRasterHasGs(false), m_activeStageCount(0) { const Vkgc::BinaryData *gpurtShaderLibrary = nullptr; #if LLPC_CLIENT_INTERFACE_MAJOR_VERSION < 62 gpurtShaderLibrary = &pipelineInfo->shaderLibrary; @@ -311,6 +311,7 @@ void GraphicsContext::setColorExportState(Pipeline *pipeline, Util::MetroHash64 state.alphaToCoverageEnable = cbState.alphaToCoverageEnable; state.dualSourceBlendEnable = cbState.dualSourceBlendEnable; + state.dualSourceBlendDynamicEnable = cbState.dualSourceBlendDynamic; for (unsigned targetIndex = 0; targetIndex < MaxColorTargets; ++targetIndex) { if (cbState.target[targetIndex].format != VK_FORMAT_UNDEFINED) { diff --git a/llpc/context/llpcGraphicsContext.h b/llpc/context/llpcGraphicsContext.h index badda9cca5..968e6d0ca3 100644 --- a/llpc/context/llpcGraphicsContext.h +++ b/llpc/context/llpcGraphicsContext.h @@ -58,12 +58,6 @@ class GraphicsContext : public PipelineContext { // Sets the mask of active shader stages bound to this pipeline virtual void setShaderStageMask(unsigned mask) override { m_stageMask = mask; } - // Sets whether dual source blend is used in fragment shader - virtual void setUseDualSourceBlend(bool useDualSourceBlend) override { m_useDualSourceBlend = useDualSourceBlend; } - - // Gets whether dual source blend is used in fragment shader - virtual bool getUseDualSourceBlend() const override { return m_useDualSourceBlend; } - // Sets whether pre-rasterization part has a geometry shader virtual void setPreRasterHasGs(bool preRasterHasGs) override { m_preRasterHasGs = preRasterHasGs; } @@ -110,7 +104,6 @@ class GraphicsContext : public PipelineContext { unsigned m_stageMask; // Mask of active shader stages bound to this graphics pipeline bool m_preRasterHasGs; // Whether pre-rasterization part has a geometry shader - bool m_useDualSourceBlend; // Whether dual source blend is used in fragment shader unsigned m_activeStageCount; // Count of active shader stages }; diff --git a/llpc/context/llpcPipelineContext.h b/llpc/context/llpcPipelineContext.h index e5b2b794dc..8087285c05 100644 --- a/llpc/context/llpcPipelineContext.h +++ b/llpc/context/llpcPipelineContext.h @@ -133,14 +133,6 @@ class PipelineContext { // Sets the mask of active shader stages bound to this pipeline virtual void setShaderStageMask(unsigned mask) = 0; - // Sets whether dual source blend is used in fragment shader - // NOTE: Only applicable in the part pipeline compilation mode. - virtual void setUseDualSourceBlend(bool useDualSourceBlend) { llvm_unreachable("Should never be called!"); } - - // Gets whether dual source blend is used in fragment shader - // NOTE: Only applicable in the part pipeline compilation mode. - virtual bool getUseDualSourceBlend() const { return false; } - // Sets whether pre-rasterization part has a geometry shader. // NOTE: Only applicable in the part pipeline compilation mode. virtual void setPreRasterHasGs(bool preRasterHasGs) { llvm_unreachable("Should never be called!"); } diff --git a/llpc/include/llpc.h b/llpc/include/llpc.h index 358f1e5e09..60735d54f2 100644 --- a/llpc/include/llpc.h +++ b/llpc/include/llpc.h @@ -103,7 +103,8 @@ static const char VkIcdName[] = "amdvlk"; /// Represents per shader module options. struct ShaderModuleOptions { PipelineOptions pipelineOptions; ///< Pipeline options related with this shader module - bool mergeLocationAndBinding; + bool mergeLocationAndBinding; ///< Merge location and binding if binding decoration is not found + unsigned resourceBindingOffset; ///< Recalculate resource bindings for each program in separate shader object mode }; /// Represents info to build a shader module. diff --git a/llpc/lower/LowerGLCompatibility.cpp b/llpc/lower/LowerGLCompatibility.cpp index badffab8d9..5a2c2cad54 100644 --- a/llpc/lower/LowerGLCompatibility.cpp +++ b/llpc/lower/LowerGLCompatibility.cpp @@ -46,7 +46,7 @@ namespace Llpc { // ===================================================================================================================== LowerGLCompatibility::LowerGLCompatibility() - : m_retInst(nullptr), m_clipVertex(nullptr), m_clipDistance(nullptr), m_clipPlane(nullptr) { + : m_retInst(nullptr), m_out(nullptr), m_clipVertex(nullptr), m_clipDistance(nullptr), m_clipPlane(nullptr) { } // ===================================================================================================================== @@ -98,6 +98,52 @@ unsigned LowerGLCompatibility::getUniformLocation(llvm::GlobalVariable *var) { return mdconst::dyn_extract(metaNode->getOperand(3))->getZExtValue(); } +// ===================================================================================================================== +// Get in/out meta data by indices from from aggregate type. +// +// @param [in] valueTy : The metadata's embellish type. +// @param [in] mds : The metadata constant of InOut Global variable to be decode. +// @param [in] index : The the index of the metadata in the embellish type. +// @param [out] out : Use to output the element's metadatas of the InOut Global variable. +void LowerGLCompatibility::decodeInOutMetaRecursivelyByIndex(llvm::Type *valueTy, llvm::Constant *mds, + ArrayRef index, + llvm::SmallVector &out) { + auto currentType = valueTy; + auto currentMds = mds; + if (!index.empty()) { + if (valueTy->isSingleValueType()) { + // Single type's metadata:{uint64, uint64} + assert(mds->getType() == StructType::get(*m_context, {m_builder->getInt64Ty(), m_builder->getInt64Ty()})); + ShaderInOutMetadata md = {}; + md.U64All[0] = cast(mds->getOperand(0))->getZExtValue(); + md.U64All[1] = cast(mds->getOperand(1))->getZExtValue(); + out.push_back(md); + } else if (valueTy->isArrayTy()) { + assert(mds->getType()->getStructNumElements() == 4); + currentType = valueTy->getArrayElementType(); + currentMds = cast(mds->getOperand(1)); + index = index.drop_front(); + if (index.empty()) + decodeInOutMetaRecursively(currentType, currentMds, out); + else { + decodeInOutMetaRecursivelyByIndex(currentType, currentMds, index, out); + } + } else if (valueTy->isStructTy()) { + // Structure type's metadata:[{element metadata type}, ...] + assert(valueTy->getStructNumElements() == mds->getType()->getStructNumElements()); + auto opIdx = cast(index[0])->getZExtValue(); + currentType = valueTy->getStructElementType(opIdx); + currentMds = cast(mds->getOperand(opIdx)); + index = index.drop_front(); + if (index.empty()) + decodeInOutMetaRecursively(currentType, currentMds, out); + else { + decodeInOutMetaRecursivelyByIndex(currentType, currentMds, index, out); + } + } + } +} + // ===================================================================================================================== // Get in/out meta data recursively. // @@ -194,13 +240,65 @@ void LowerGLCompatibility::collectEmulationResource() { MDNode *metaNode = global.getMetadata(gSPIRVMD::InOut); assert(metaNode); auto inOutMetaConst = mdconst::dyn_extract(metaNode->getOperand(0)); - decodeInOutMetaRecursively(global.getValueType(), inOutMetaConst, mds); + auto valueType = global.getValueType(); + bool isStructureOrArrayOfStructure = + (valueType->isStructTy() || (valueType->isArrayTy() && valueType->getArrayElementType()->isStructTy())); + decodeInOutMetaRecursively(valueType, inOutMetaConst, mds); for (auto md : mds) { if (md.IsLoc) { - if (md.Value == Vkgc::GlCompatibilityInOutLocation::ClipVertex) - m_clipVertex = &global; + if (md.Value == Vkgc::GlCompatibilityInOutLocation::ClipVertex) { + if (isStructureOrArrayOfStructure) + m_out = &global; + else + m_clipVertex = &global; + } } else if (md.IsBuiltIn && md.Value == spv::BuiltInClipDistance) { - m_clipDistance = &global; + if (isStructureOrArrayOfStructure) + m_out = &global; + else + m_clipDistance = &global; + } + } + } + } + + // If gl_in/gl_out used in shader, then the Gl deprecated builtin variable will be pack in the structure: + // gl_PerVertex. We need traversal the user of m_out to get the usage information Gl deprecated builtin variable. + if (m_out != nullptr) { + assert((m_clipVertex == nullptr) && (m_clipDistance == nullptr)); + llvm::SmallVector mds; + auto glOut = cast(m_out); + MDNode *metaNode = glOut->getMetadata(gSPIRVMD::InOut); + assert(metaNode); + auto inOutMetaConst = mdconst::dyn_extract(metaNode->getOperand(0)); + for (User *user : m_out->users()) { + if (GetElementPtrInst *gep = dyn_cast(user)) { + // The user is a GEP + // Check to see if the value has been stored. + bool beenModified = false; + for (User *gepUser : gep->users()) { + assert(!isa(gepUser)); + beenModified |= isa(gepUser); + } + + // We shouldn't have any chained GEPs here, they are coalesced by the LowerAccessChain pass. + SmallVector indexOperands; + for (auto index = gep->idx_begin(); index != gep->idx_end(); index++) { + // Skip the first indices, it should be 0 in most of time. + if (index == gep->idx_begin()) { + assert(cast(gep->idx_begin())->isZero() && "Non-zero GEP first index\n"); + continue; + } + indexOperands.push_back(m_builder->CreateZExtOrTrunc(index->get(), m_builder->getInt32Ty())); + } + decodeInOutMetaRecursivelyByIndex(glOut->getValueType(), inOutMetaConst, indexOperands, mds); + for (auto md : mds) { + if (md.IsLoc) { + if (beenModified && (md.Value == Vkgc::GlCompatibilityInOutLocation::ClipVertex)) + m_clipVertex = gep; + } else if (md.IsBuiltIn && md.Value == spv::BuiltInClipDistance) { + m_clipDistance = gep; + } } } } @@ -226,13 +324,18 @@ bool LowerGLCompatibility::needLowerClipVertex() { // Create the SPIR-V output builtin variable "ClipDistance". void LowerGLCompatibility::createClipDistance() { assert(m_clipDistance == nullptr); + auto *buildInfo = static_cast(m_context->getPipelineBuildInfo()); + uint32_t indexOfLastClipPlane = 0; + Util::BitMaskScanReverse(&indexOfLastClipPlane, buildInfo->rsState.usrClipPlaneMask); + auto floatType = m_builder->getFloatTy(); auto int32Type = m_builder->getInt32Ty(); auto int64Type = m_builder->getInt64Ty(); - auto clipDistanceType = ArrayType::get(floatType, 8); - m_clipDistance = new GlobalVariable(*m_module, clipDistanceType, false, GlobalValue::ExternalLinkage, nullptr, - "gl_ClipDistance", nullptr, GlobalVariable::NotThreadLocal, SPIRV::SPIRAS_Output); + auto clipDistanceType = ArrayType::get(floatType, indexOfLastClipPlane + 1); + auto clipDistance = + new GlobalVariable(*m_module, clipDistanceType, false, GlobalValue::ExternalLinkage, nullptr, "gl_ClipDistance", + nullptr, GlobalVariable::NotThreadLocal, SPIRV::SPIRAS_Output); ShaderInOutMetadata inOutMd = {}; inOutMd.IsBuiltIn = true; @@ -267,7 +370,8 @@ void LowerGLCompatibility::createClipDistance() { std::vector mDs; mDs.push_back(ConstantAsMetadata::get(mdVariable)); auto mdNode = MDNode::get(*m_context, mDs); - m_clipDistance->addMetadata(gSPIRVMD::InOut, *mdNode); + clipDistance->addMetadata(gSPIRVMD::InOut, *mdNode); + m_clipDistance = clipDistance; } // ===================================================================================================================== @@ -276,7 +380,7 @@ void LowerGLCompatibility::createClipPlane() { auto floatType = m_builder->getFloatTy(); auto vec4Type = FixedVectorType::get(floatType, 4); auto clipPlaneType = ArrayType::get(vec4Type, 8); - m_clipPlane = + auto clipPlane = new GlobalVariable(*m_module, clipPlaneType, false, GlobalValue::ExternalLinkage, nullptr, "gl_ClipPlaneInternal", nullptr, GlobalVariable::NotThreadLocal, SPIRV::SPIRAS_Uniform); auto locationFound = @@ -295,7 +399,8 @@ void LowerGLCompatibility::createClipPlane() { mDs.push_back(ConstantAsMetadata::get(ConstantInt::get(int32Ty, clipPlaneBaseOffset))); mDs.push_back(ConstantAsMetadata::get(ConstantInt::get(int32Ty, Vkgc::GlCompatibilityUniformLocation::ClipPlane))); auto mdNode = MDNode::get(*m_context, mDs); - m_clipPlane->addMetadata(gSPIRVMD::UniformConstant, *mdNode); + clipPlane->addMetadata(gSPIRVMD::UniformConstant, *mdNode); + m_clipPlane = clipPlane; } // ===================================================================================================================== @@ -304,7 +409,7 @@ void LowerGLCompatibility::emulateStoreClipVertex() { auto floatType = m_builder->getFloatTy(); Type *vec4Type = VectorType::get(floatType, 4, false); // Load clipVertex - auto clipVertex = m_builder->CreateLoad(m_clipVertex->getValueType(), m_clipVertex); + Value *clipVertex = m_builder->CreateLoad(vec4Type, m_clipVertex); // Create a new intermediate result variable assert(m_context->getPipelineType() == PipelineType::Graphics); auto *buildInfo = static_cast(m_context->getPipelineBuildInfo()); @@ -312,16 +417,14 @@ void LowerGLCompatibility::emulateStoreClipVertex() { for (uint32_t clipPlaneIdx = 0; clipPlaneIdx < Vkgc::GlCompatibilityLimits::MaxClipPlanes; ++clipPlaneIdx) { if (clipPlaneMask & (1 << clipPlaneIdx)) { // gl_ClipPlane are Emulate by uniform constant, so the resource descriptor are same with uniform constant. - auto clipPlaneElement = m_builder->CreateConstInBoundsGEP1_32(m_clipPlane->getValueType()->getArrayElementType(), - m_clipPlane, clipPlaneIdx); + auto clipPlaneElement = m_builder->CreateConstInBoundsGEP1_32(vec4Type, m_clipPlane, clipPlaneIdx); auto clipPlaneLoad = m_builder->CreateLoad(vec4Type, clipPlaneElement); // Dot ClipPlane and ClipVertex auto dot = m_builder->CreateDotProduct(clipVertex, clipPlaneLoad); // Store result to ClipDistance - auto clipDistanceElement = m_builder->CreateConstInBoundsGEP1_32( - m_clipDistance->getValueType()->getArrayElementType(), m_clipDistance, clipPlaneIdx); + auto clipDistanceElement = m_builder->CreateConstInBoundsGEP1_32(floatType, m_clipDistance, clipPlaneIdx); m_builder->CreateStore(dot, clipDistanceElement); } } diff --git a/llpc/lower/LowerGLCompatibility.h b/llpc/lower/LowerGLCompatibility.h index d7786dda05..e8fde951c7 100644 --- a/llpc/lower/LowerGLCompatibility.h +++ b/llpc/lower/LowerGLCompatibility.h @@ -47,6 +47,8 @@ class LowerGLCompatibility : public SpirvLower, public llvm::PassInfoMixin index, + llvm::SmallVector &out); void decodeInOutMetaRecursively(llvm::Type *valueTy, llvm::Constant *mds, llvm::SmallVector &out); void unifyFunctionReturn(Function *func); @@ -65,9 +67,10 @@ class LowerGLCompatibility : public SpirvLower, public llvm::PassInfoMixingetPipelineType() == PipelineType::RayTracing); + auto *pipelineInfo = static_cast(context->getPipelineBuildInfo()); + if (pipelineInfo->mode != Vkgc::LlpcRaytracingMode::Continuations) { + passMgr.addPass(SpirvLowerRayTracing()); + passMgr.addPass(AlwaysInlinerPass()); + } + } + if (rayTracing || rayQuery || isInternalRtShader) { FunctionPassManager fpm; fpm.addPass(SROAPass(SROAOptions::PreserveCFG)); @@ -320,12 +324,14 @@ void SpirvLower::replaceGlobal(Context *context, GlobalVariable *original, Globa void SpirvLower::init(Module *module) { m_module = module; m_context = static_cast(&m_module->getContext()); - if (m_module->empty()) { + SmallVector entries; + getEntryPoints(module, entries); + if (entries.size() != 1) { m_shaderStage = ShaderStageInvalid; m_entryPoint = nullptr; } else { - m_shaderStage = getShaderStageFromModule(m_module); - m_entryPoint = getEntryPoint(m_module); + m_entryPoint = entries[0]; + m_shaderStage = getShaderStageFromFunction(m_entryPoint); if (m_shaderStage == ShaderStageInvalid) { // There might be cases we fail to get shader stage from a module that is not directly converted from SPIR-V, for // example, unified ray tracing pipeline shader, or entry for indirect ray tracing pipeline. In such case, clamp diff --git a/llpc/lower/llpcSpirvLowerGlobal.cpp b/llpc/lower/llpcSpirvLowerGlobal.cpp index 5a6ace802b..d42db9252c 100644 --- a/llpc/lower/llpcSpirvLowerGlobal.cpp +++ b/llpc/lower/llpcSpirvLowerGlobal.cpp @@ -590,6 +590,7 @@ void SpirvLowerGlobal::mapGlobalVariableToProxy(GlobalVariable *globalVar) { const auto &dataLayout = m_module->getDataLayout(); Type *globalVarTy = globalVar->getValueType(); + assert(m_entryPoint); Value *proxy = nullptr; assert(m_entryPoint); removeConstantExpr(m_context, globalVar); @@ -1216,11 +1217,16 @@ Value *SpirvLowerGlobal::addCallInstForInOutImport(Type *inOutTy, unsigned addrS } inOutInfo.setPerPrimitive(inOutMeta.PerPrimitive); - if (addrSpace == SPIRAS_Input) - inOutValue = m_builder->CreateReadBuiltInInput(builtIn, inOutInfo, vertexIdx, elemIdx); - else + if (addrSpace == SPIRAS_Input) { + // In the case where the command has no baseVertex parameter, force the value of gl_BaseVertex to zero + if (builtIn == lgc::BuiltInBaseVertex && + m_context->getPipelineContext()->getPipelineOptions()->disableBaseVertex) + inOutValue = m_builder->getInt32(0); + else + inOutValue = m_builder->CreateReadBuiltInInput(builtIn, inOutInfo, vertexIdx, elemIdx); + } else { inOutValue = m_builder->CreateReadBuiltInOutput(builtIn, inOutInfo, vertexIdx, elemIdx); - + } if ((builtIn == lgc::BuiltInSubgroupEqMask || builtIn == lgc::BuiltInSubgroupGeMask || builtIn == lgc::BuiltInSubgroupGtMask || builtIn == lgc::BuiltInSubgroupLeMask || builtIn == lgc::BuiltInSubgroupLtMask) && @@ -1469,9 +1475,6 @@ void SpirvLowerGlobal::addCallInstForOutputExport(Value *outputValue, Constant * cast(locOffset)->getZExtValue(), outputInfo); } - if (m_context->getPipelineContext()->getUseDualSourceBlend()) { - outputInfo.setDualSourceBlendDynamic(true); - } m_builder->CreateWriteGenericOutput(outputValue, location, locOffset, elemIdx, maxLocOffset, outputInfo, vertexOrPrimitiveIdx); } diff --git a/llpc/lower/llpcSpirvLowerInternalLibraryIntrinsicUtil.cpp b/llpc/lower/llpcSpirvLowerInternalLibraryIntrinsicUtil.cpp index 2c421ab97b..620c745f3c 100644 --- a/llpc/lower/llpcSpirvLowerInternalLibraryIntrinsicUtil.cpp +++ b/llpc/lower/llpcSpirvLowerInternalLibraryIntrinsicUtil.cpp @@ -30,6 +30,7 @@ */ #include "llpcSpirvLowerInternalLibraryIntrinsicUtil.h" +#include "SPIRVInternal.h" #include "llpcContext.h" #include "lgc/Builder.h" #include "llvm/IR/IntrinsicsAMDGPU.h" @@ -45,7 +46,7 @@ namespace Llpc { // @param func : The function to process // @param builder : The IR builder static void createLaneIndex(Function *func, Builder *builder) { - builder->CreateRet(builder->CreateReadBuiltInInput(static_cast(BuiltInSubgroupLocalInvocationId))); + builder->CreateRet(builder->CreateReadBuiltInInput(lgc::BuiltInSubgroupLocalInvocationId)); } // ===================================================================================================================== @@ -67,12 +68,316 @@ static void createHalt(Function *func, Builder *builder) { builder->CreateRetVoid(); } +// ===================================================================================================================== +// Create function to compute the number of waves in the workgroup +// +// @param func : The function to process +// @param builder : The IR builder +static void createNumWavesCompute(Function *func, Builder *builder) { + Value *workgroupSize = builder->CreateReadBuiltInInput(lgc::BuiltInWorkgroupSize, {}, nullptr, nullptr, ""); + Value *workgroupSizeX = builder->CreateExtractElement(workgroupSize, uint64_t(0)); + Value *workgroupSizeY = builder->CreateExtractElement(workgroupSize, 1); + Value *workgroupSizeZ = builder->CreateExtractElement(workgroupSize, 2); + Value *numWaves = builder->CreateMul(workgroupSizeX, workgroupSizeY); + numWaves = builder->CreateMul(numWaves, workgroupSizeZ); + numWaves = builder->CreateSub(numWaves, builder->getInt32(1)); + Value *waveSize = builder->CreateGetWaveSize(); + numWaves = builder->CreateAdd(numWaves, waveSize); + numWaves = builder->CreateUDiv(numWaves, waveSize); + builder->CreateRet(numWaves); +} + +// ===================================================================================================================== +// Create function to compute the wave index in the workgroup +// +// @param func : The function to process +// @param builder : The IR builder +static void createWaveIndexCompute(Function *func, Builder *builder) { + // return (gl_LocalInvocationIndex / WaveSize) + Value *flattenedThreadId = + builder->CreateReadBuiltInInput(lgc::BuiltInLocalInvocationIndex, {}, nullptr, nullptr, ""); + Value *waveSize = builder->CreateGetWaveSize(); + builder->CreateRet(builder->CreateUDiv(flattenedThreadId, waveSize)); +} + +// ===================================================================================================================== +// Create function to get gl_WorkGroupID +// +// @param func : The function to process +// @param builder : The IR builder +static void createGroupIdCompute(Function *func, Builder *builder) { + Value *workGroupId = builder->CreateReadBuiltInInput(lgc::BuiltInWorkgroupId, {}, nullptr, nullptr, ""); + builder->CreateRet(workGroupId); +} + +// ===================================================================================================================== +// Create function to get gl_WorkGroupSize +// +// @param func : The function to process +// @param builder : The IR builder +static void createGroupDimCompute(Function *func, Builder *builder) { + Value *workGroupSize = builder->CreateReadBuiltInInput(lgc::BuiltInWorkgroupSize, {}, nullptr, nullptr, ""); + builder->CreateRet(workGroupSize); +} + +// ===================================================================================================================== +// Create gl_LocalInvocationID +// +// @param func : The function to process +// @param builder : The IR builder +static void createThreadIdInGroupCompute(Function *func, Builder *builder) { + Value *threadId = builder->CreateReadBuiltInInput(lgc::BuiltInLocalInvocationId, {}, nullptr, nullptr, ""); + builder->CreateRet(threadId); +} + +// ===================================================================================================================== +// Create gl_LocalInvocationIndex, return uint +// +// @param func : The function to process +// @param builder : The IR builder +static void createFlattenedThreadIdInGroupCompute(Function *func, Builder *builder) { + Value *threadId = builder->CreateReadBuiltInInput(lgc::BuiltInLocalInvocationIndex, {}, nullptr, nullptr, ""); + builder->CreateRet(threadId); +} + +// ===================================================================================================================== +// Create subgroup mbcnt function +// +// @param func : The function to process +// @param builder : The IR builder +static void createMbcnt(Function *func, Builder *builder) { + Value *ptr = func->getArg(0); + Value *mask = builder->CreateLoad(FixedVectorType::get(builder->getInt32Ty(), 2), ptr); + mask = builder->CreateBitCast(mask, builder->getInt64Ty()); + builder->CreateRet(builder->CreateSubgroupMbcnt(mask)); +} + +// ===================================================================================================================== +// Create atomic function +// +// @param func : The function to process +// @param builder : The IR builder +// @param is64 : Whether we're creating a 64-bit atomic +// @param isCmpXchg : Whether we're creating a cmpxchg atomic +// @param binOp : If we're not creating a cmpxchg atomic, then this is the RMW atomic op +static void createAtomic(Function *func, Builder *builder, bool is64, bool isCmpXchg, + AtomicRMWInst::BinOp binOp = AtomicRMWInst::BAD_BINOP) { + auto argIt = func->arg_begin(); + Value *arg = argIt++; + Type *gpuVaTy = builder->getInt64Ty(); + Value *gpuAddr = builder->CreateLoad(gpuVaTy, arg); + arg = argIt++; + Type *offsetTy = builder->getInt32Ty(); + Value *offset = builder->CreateLoad(offsetTy, arg); + Type *valueTy = is64 ? builder->getInt64Ty() : builder->getInt32Ty(); + Value *compare = isCmpXchg ? builder->CreateLoad(valueTy, argIt++) : nullptr; + arg = argIt; + Value *value = builder->CreateLoad(valueTy, arg); + Type *gpuAddrAsPtrTy = PointerType::get(builder->getContext(), SPIRAS_Global); + auto gpuAddrAsPtr = builder->CreateIntToPtr(gpuAddr, gpuAddrAsPtrTy); + // Create GEP to get the byte address with byte offset + gpuAddrAsPtr = builder->CreateGEP(builder->getInt8Ty(), gpuAddrAsPtr, offset); + Value *atomicValue = nullptr; + if (!isCmpXchg) { + assert(binOp != AtomicRMWInst::BAD_BINOP); + atomicValue = builder->CreateAtomicRMW(binOp, gpuAddrAsPtr, value, MaybeAlign(), AtomicOrdering::Monotonic, + SyncScope::System); + } else { + atomicValue = builder->CreateAtomicCmpXchg(gpuAddrAsPtr, compare, value, MaybeAlign(), AtomicOrdering::Monotonic, + AtomicOrdering::Monotonic, SyncScope::System); + atomicValue = builder->CreateExtractValue(atomicValue, 0); + } + builder->CreateRet(atomicValue); +} + +// ===================================================================================================================== +// Create 32-bit atomic add at address +// +// @param func : The function to process +// @param builder : The IR builder +static void createAtomicAddAtAddr(Function *func, Builder *builder) { + createAtomic(func, builder, /*is64*/ false, /*isCmpXchg*/ false, AtomicRMWInst::Add); +} + +// ===================================================================================================================== +// Create 32-bit atomic max at address +// +// @param func : The function to process +// @param builder : The IR builder +static void createAtomicMaxAtAddr(Function *func, Builder *builder) { + createAtomic(func, builder, /*is64*/ false, /*isCmpXchg*/ false, AtomicRMWInst::Max); +} + +// ===================================================================================================================== +// Create 32-bit atomic and at address +// +// @param func : The function to process +// @param builder : The IR builder +static void createAtomicAndAtAddr(Function *func, Builder *builder) { + createAtomic(func, builder, /*is64*/ false, /*isCmpXchg*/ false, AtomicRMWInst::And); +} + +// ===================================================================================================================== +// Create 32-bit atomic or at address +// +// @param func : The function to process +// @param builder : The IR builder +static void createAtomicOrAtAddr(Function *func, Builder *builder) { + createAtomic(func, builder, /*is64*/ false, /*isCmpXchg*/ false, AtomicRMWInst::Or); +} + +// ===================================================================================================================== +// Create 64-bit atomic add at address +// +// @param func : The function to process +// @param builder : The IR builder +static void createAtomic64AddAtAddr(Function *func, Builder *builder) { + createAtomic(func, builder, /*is64*/ true, /*isCmpXchg*/ false, AtomicRMWInst::Add); +} + +// ===================================================================================================================== +// Create 64-bit atomic max at address +// +// @param func : The function to process +// @param builder : The IR builder +static void createAtomic64MaxAtAddr(Function *func, Builder *builder) { + createAtomic(func, builder, /*is64*/ true, /*isCmpXchg*/ false, AtomicRMWInst::Max); +} + +// ===================================================================================================================== +// Create 64-bit atomic and at address +// +// @param func : The function to process +// @param builder : The IR builder +static void createAtomic64AndAtAddr(Function *func, Builder *builder) { + createAtomic(func, builder, /*is64*/ true, /*isCmpXchg*/ false, AtomicRMWInst::And); +} + +// ===================================================================================================================== +// Create 64-bit atomic or at address +// +// @param func : The function to process +// @param builder : The IR builder +static void createAtomic64OrAtAddr(Function *func, Builder *builder) { + createAtomic(func, builder, /*is64*/ true, /*isCmpXchg*/ false, AtomicRMWInst::Or); +} + +// ===================================================================================================================== +// Create 64-bit atomic exchange at address +// +// @param func : The function to process +// @param builder : The IR builder +static void createAtomic64XchgAtAddr(Function *func, Builder *builder) { + createAtomic(func, builder, /*is64*/ true, /*isCmpXchg*/ false, AtomicRMWInst::Xchg); +} + +// ===================================================================================================================== +// Create 64-bit atomic compare and exchange at address +// +// @param func : The function to process +// @param builder : The IR builder +static void createAtomic64CmpXchgAtAddr(Function *func, Builder *builder) { + createAtomic(func, builder, /*is64*/ true, /*isCmpXchg*/ true); +} + +// ===================================================================================================================== +// Create load store function cached/unchached +// +// @param func : The function to process +// @param builder : The IR builder +static void createLoadStore(Function *func, Builder *builder, bool isLoad, bool isUncached) { + auto argIt = func->arg_begin(); + Value *arg = argIt++; + Type *gpuVaTy = builder->getInt64Ty(); + Value *gpuAddr = builder->CreateLoad(gpuVaTy, arg); + arg = argIt; + Type *offsetTy = builder->getInt32Ty(); + Value *offset = builder->CreateLoad(offsetTy, arg); + Type *gpuAddrAsPtrTy = PointerType::get(builder->getContext(), SPIRAS_Global); + Value *gpuAddrAsPtr = builder->CreateIntToPtr(gpuAddr, gpuAddrAsPtrTy); + gpuAddrAsPtr = builder->CreateGEP(builder->getInt8Ty(), gpuAddrAsPtr, offset); + + // Cast to the return type pointer + Type *gpuAddrAsTy = builder->getInt32Ty(); + gpuAddrAsPtrTy = gpuAddrAsTy->getPointerTo(SPIRAS_Global); + gpuAddrAsPtr = builder->CreateBitCast(gpuAddrAsPtr, gpuAddrAsPtrTy); + + // Load value + if (isLoad) { + Value *loadValue = builder->CreateLoad(gpuAddrAsTy, gpuAddrAsPtr, isUncached); + builder->CreateRet(loadValue); + } else { + arg = ++argIt; + Type *valueTy = builder->getInt32Ty(); + Value *data = builder->CreateLoad(valueTy, arg); + builder->CreateStore(data, gpuAddrAsPtr, isUncached); + builder->CreateRetVoid(); + } +} + +// ===================================================================================================================== +// Create load dword at address +// +// @param func : The function to process +// @param builder : The IR builder +static void createLoadDwordAtAddr(Function *func, Builder *builder) { + createLoadStore(func, builder, true, false); +} + +// ===================================================================================================================== +// Create load uncached dword at address +// +// @param func : The function to process +// @param builder : The IR builder +static void createLoadDwordAtAddrUncached(Function *func, Builder *builder) { + createLoadStore(func, builder, true, true); +} + +// ===================================================================================================================== +// Create store dword at address +// +// @param func : The function to process +// @param builder : The IR builder +static void createStoreDwordAtAddr(Function *func, Builder *builder) { + createLoadStore(func, builder, false, false); +} + +// ===================================================================================================================== +// Create store uncached dword at address +// +// @param func : The function to process +// @param builder : The IR builder +static void createStoreDwordAtAddrUncached(Function *func, Builder *builder) { + createLoadStore(func, builder, false, true); +} + // ===================================================================================================================== // Initialize library function pointer table InternalLibraryIntrinsicUtil::LibraryFunctionTable::LibraryFunctionTable() { m_libFuncPtrs["AmdExtLaneIndex"] = &createLaneIndex; m_libFuncPtrs["AmdExtLaneCount"] = &createLaneCount; m_libFuncPtrs["AmdExtHalt"] = &createHalt; + m_libFuncPtrs["AmdExtNumWavesCompute"] = &createNumWavesCompute; + m_libFuncPtrs["AmdExtWaveIndexCompute"] = &createWaveIndexCompute; + m_libFuncPtrs["AmdExtGroupIdCompute"] = &createGroupIdCompute; + m_libFuncPtrs["AmdExtGroupDimCompute"] = &createGroupDimCompute; + m_libFuncPtrs["AmdExtThreadIdInGroupCompute"] = &createThreadIdInGroupCompute; + m_libFuncPtrs["AmdExtFlattenedThreadIdInGroupCompute"] = &createFlattenedThreadIdInGroupCompute; + m_libFuncPtrs["AmdExtMbcnt"] = &createMbcnt; + m_libFuncPtrs["AmdExtAtomicAddAtAddr"] = &createAtomicAddAtAddr; + m_libFuncPtrs["AmdExtAtomicMaxAtAddr"] = &createAtomicMaxAtAddr; + m_libFuncPtrs["AmdExtAtomicAndAtAddr"] = &createAtomicAndAtAddr; + m_libFuncPtrs["AmdExtAtomicOrAtAddr"] = &createAtomicOrAtAddr; + m_libFuncPtrs["AmdExtAtomic64AddAtAddr"] = &createAtomic64AddAtAddr; + m_libFuncPtrs["AmdExtAtomic64MaxAtAddr"] = &createAtomic64MaxAtAddr; + m_libFuncPtrs["AmdExtAtomic64AndAtAddr"] = &createAtomic64AndAtAddr; + m_libFuncPtrs["AmdExtAtomic64OrAtAddr"] = &createAtomic64OrAtAddr; + m_libFuncPtrs["AmdExtAtomic64XchgAtAddr"] = &createAtomic64XchgAtAddr; + m_libFuncPtrs["AmdExtAtomic64CmpXchgAtAddr"] = &createAtomic64CmpXchgAtAddr; + m_libFuncPtrs["AmdExtLoadDwordAtAddr"] = &createLoadDwordAtAddr; + m_libFuncPtrs["AmdExtLoadDwordAtAddrUncached"] = &createLoadDwordAtAddrUncached; + m_libFuncPtrs["AmdExtStoreDwordAtAddr"] = &createStoreDwordAtAddr; + m_libFuncPtrs["AmdExtStoreDwordAtAddrUncached"] = &createStoreDwordAtAddrUncached; } } // namespace Llpc diff --git a/llpc/lower/llpcSpirvLowerRayQueryPostInline.cpp b/llpc/lower/llpcSpirvLowerRayQueryPostInline.cpp index 335ea532e3..2d1cc2ba08 100644 --- a/llpc/lower/llpcSpirvLowerRayQueryPostInline.cpp +++ b/llpc/lower/llpcSpirvLowerRayQueryPostInline.cpp @@ -71,7 +71,8 @@ bool SpirvLowerRayQueryPostInline::runImpl(Module &module) { for (auto funcIt = module.begin(), funcEnd = module.end(); funcIt != funcEnd;) { Function *func = &*funcIt++; - if (func->getLinkage() == GlobalValue::ExternalLinkage && !func->empty()) { + if ((func->getLinkage() == GlobalValue::ExternalLinkage || func->getLinkage() == GlobalValue::WeakAnyLinkage) && + !func->empty()) { if (!func->getName().startswith(m_entryPoint->getName())) { func->dropAllReferences(); func->eraseFromParent(); diff --git a/llpc/lower/llpcSpirvLowerRayTracing.cpp b/llpc/lower/llpcSpirvLowerRayTracing.cpp index 4c70f17c97..19a830f33b 100644 --- a/llpc/lower/llpcSpirvLowerRayTracing.cpp +++ b/llpc/lower/llpcSpirvLowerRayTracing.cpp @@ -654,13 +654,15 @@ PreservedAnalyses SpirvLowerRayTracing::run(Module &module, ModuleAnalysisManage func->eraseFromParent(); } - // Newly generated implementation functions are external linkage, fix that. for (auto funcIt = module.begin(), funcEnd = module.end(); funcIt != funcEnd;) { Function *func = &*funcIt++; - if (func->getLinkage() == GlobalValue::ExternalLinkage && !func->empty()) { - if (!func->getName().startswith(module.getName())) { - func->setLinkage(GlobalValue::InternalLinkage); - } + if (!func->empty() && !func->getName().startswith(module.getName()) && + ((func->getLinkage() == GlobalValue::ExternalLinkage) || (func->getLinkage() == GlobalValue::WeakAnyLinkage))) { + // Newly generated implementation functions have external linkage, but should have internal linkage. + // Weak-linkage functions are GpuRt functions that we just added calls to, and which are no longer required apart + // from these calls, so assign internal linkage to them as well. + // In both cases, these functions are removed after inlining. + func->setLinkage(GlobalValue::InternalLinkage); } } diff --git a/llpc/lower/llpcSpirvLowerUtil.cpp b/llpc/lower/llpcSpirvLowerUtil.cpp index 05261490ad..b055eefcce 100644 --- a/llpc/lower/llpcSpirvLowerUtil.cpp +++ b/llpc/lower/llpcSpirvLowerUtil.cpp @@ -41,38 +41,66 @@ using namespace llvm; namespace Llpc { // ===================================================================================================================== -// Gets the entry point (valid for AMD GPU) of a LLVM module. +// Gets all entry points of a LLVM module. // // @param module : LLVM module -Function *getEntryPoint(Module *module) { - Function *entryPoint = nullptr; - +// @param result : Vector that all entries are appended to. +void getEntryPoints(Module *module, SmallVectorImpl &result) { for (auto func = module->begin(), end = module->end(); func != end; ++func) { if (!func->empty() && func->getLinkage() == GlobalValue::ExternalLinkage) { - entryPoint = &*func; - break; + result.push_back(&*func); } } - - assert(entryPoint); - return entryPoint; } // ===================================================================================================================== -// Gets the shader stage from the specified single-shader LLVM module. +// Gets the unique entry point (valid for AMD GPU) of a LLVM module. // // @param module : LLVM module -ShaderStage getShaderStageFromModule(Module *module) { - Function *func = getEntryPoint(module); +Function *getEntryPoint(Module *module) { + SmallVector entries; + getEntryPoints(module, entries); + assert(entries.size() == 1); + return entries[0]; +} +// ===================================================================================================================== +// Gets the shader stage from the specified single-shader LLVM function. +// +// @param module : LLVM module +ShaderStage getShaderStageFromFunction(Function *function) { // Check for the execution model metadata that is added by the SPIR-V reader. - MDNode *execModelNode = func->getMetadata(gSPIRVMD::ExecutionModel); + MDNode *execModelNode = function->getMetadata(gSPIRVMD::ExecutionModel); if (!execModelNode) return ShaderStageInvalid; + auto execModel = mdconst::dyn_extract(execModelNode->getOperand(0))->getZExtValue(); return convertToShaderStage(execModel); } +// ===================================================================================================================== +// Gets the shader stage from the specified single-shader LLVM module. +// +// @param module : LLVM module +ShaderStage getShaderStageFromModule(Module *module) { + // When processing the GpuRt module, there can initially be multiple entries, + // so we can't use getEntryPoint. + SmallVector entries; + getEntryPoints(module, entries); + + std::optional result; + + for (Function *func : entries) { + ShaderStage funcStage = getShaderStageFromFunction(func); + assert(funcStage == result.value_or(funcStage)); + result = funcStage; +#ifdef NDEBUG + break; +#endif + } + return result.value(); +} + // ===================================================================================================================== // Set the shader stage to the specified LLVM module entry function. // @@ -109,7 +137,8 @@ BasicBlock *clearBlock(Function *func) { void clearNonEntryFunctions(Module *module, StringRef entryName) { for (auto funcIt = module->begin(), funcEnd = module->end(); funcIt != funcEnd;) { Function *func = &*funcIt++; - if (func->getLinkage() == GlobalValue::ExternalLinkage && !func->empty()) { + if ((func->getLinkage() == GlobalValue::ExternalLinkage || func->getLinkage() == GlobalValue::WeakAnyLinkage) && + !func->empty()) { if (!func->getName().startswith(entryName)) { func->dropAllReferences(); func->eraseFromParent(); diff --git a/llpc/lower/llpcSpirvLowerUtil.h b/llpc/lower/llpcSpirvLowerUtil.h index d0852d2b92..eeeaeb3393 100644 --- a/llpc/lower/llpcSpirvLowerUtil.h +++ b/llpc/lower/llpcSpirvLowerUtil.h @@ -37,6 +37,7 @@ namespace llvm { class Function; class Module; class BasicBlock; +template class SmallVectorImpl; class StringRef; } // namespace llvm @@ -52,15 +53,25 @@ const static char OutputProxyPrefix[] = "__llpc_output_proxy_"; } // namespace LlpcName +// Gets the shader stage from the specified LLVM function. +ShaderStage getShaderStageFromFunction(llvm::Function *function); + // Gets the shader stage from the specified LLVM module. ShaderStage getShaderStageFromModule(llvm::Module *module); // Set the shader stage to the specified LLVM module. void setShaderStageToModule(llvm::Module *module, ShaderStage shaderStage); -// Gets the entry point (valid for AMD GPU) of a LLVM module. +// Gets the unique entry point (valid for AMD GPU) of a LLVM module. +// Asserts that there is only one. +// Entry points are determined as functions with external linkage. llvm::Function *getEntryPoint(llvm::Module *module); +// Gets all entry points in a module. Usually in LLPC we have only a single +// entry per module, but there are cases (e.g. after having imported the gpurt module) +// where this is not guaranteed. +void getEntryPoints(llvm::Module *module, llvm::SmallVectorImpl &result); + // Clears the empty block llvm::BasicBlock *clearBlock(llvm::Function *func); // Clear non entry external functions diff --git a/llpc/lower/llpcSpirvProcessGpuRtLibrary.cpp b/llpc/lower/llpcSpirvProcessGpuRtLibrary.cpp index b792e51f78..2282124c4a 100644 --- a/llpc/lower/llpcSpirvProcessGpuRtLibrary.cpp +++ b/llpc/lower/llpcSpirvProcessGpuRtLibrary.cpp @@ -142,8 +142,8 @@ void SpirvProcessGpuRtLibrary::processLibraryFunction(Function *&func) { if (funcName.startswith(traceRayFuncName) || funcName.startswith(rayQueryInitializeFuncName) || funcName.startswith(rayQueryProceedFuncName) || funcName.startswith(fetchTrianglePositionFromNodePointerFuncName) || - funcName.startswith(fetchTrianglePositionFromRayQueryFuncName)) { - func->setLinkage(GlobalValue::ExternalLinkage); + funcName.startswith(fetchTrianglePositionFromRayQueryFuncName) || funcName.startswith("_cont_")) { + func->setLinkage(GlobalValue::WeakAnyLinkage); return; } diff --git a/llpc/test/shaderdb/debug_info/avoid/DebugInfo_DebugTypedef.spvasm b/llpc/test/shaderdb/debug_info/avoid/DebugInfo_DebugTypedef.spvasm index 941d2dfacf..93ce9f96ca 100644 --- a/llpc/test/shaderdb/debug_info/avoid/DebugInfo_DebugTypedef.spvasm +++ b/llpc/test/shaderdb/debug_info/avoid/DebugInfo_DebugTypedef.spvasm @@ -19,7 +19,7 @@ OpMemoryModel Logical GLSL450 OpEntryPoint Vertex %main "main" %src = OpString "simple.hlsl" -%code = OpString "typdef float Foo; +%code = OpString "typedef float Foo; main() { Foo foo; }" diff --git a/llpc/test/shaderdb/general/PipelineVsFs_TestBarycentric_tri_fan.pipe b/llpc/test/shaderdb/general/PipelineVsFs_TestBarycentric_tri_fan.pipe index bd7b554f54..8d23a38228 100644 --- a/llpc/test/shaderdb/general/PipelineVsFs_TestBarycentric_tri_fan.pipe +++ b/llpc/test/shaderdb/general/PipelineVsFs_TestBarycentric_tri_fan.pipe @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by tool/update_llpc_test_checks.py UTC_ARGS: --function amdgpu_ps_main ; RUN: amdllpc -filetype=asm -gfxip=10.3 -o - %s | FileCheck -check-prefix=SHADERTEST %s +; REQUIRES: do-not-run ; This test tests barycentric coordinate when topology is VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN ; barycentric coordinate: (i ,j , 1 - i - j). @@ -143,21 +144,23 @@ attribute[1].format = VK_FORMAT_R32G32B32A32_SFLOAT attribute[1].offset = 16 ; SHADERTEST-LABEL: amdgpu_ps_main: -; SHADERTEST: v_mbcnt_lo_u32_b32 v2, -1, 0 -; SHADERTEST-NEXT: v_sub_f32_e32 v3, 1.0, v0 -; SHADERTEST-NEXT: v_mbcnt_hi_u32_b32 v2, -1, v2 -; SHADERTEST-NEXT: v_sub_f32_e32 v3, v3, v1 -; SHADERTEST-NEXT: v_lshrrev_b32_e32 v2, 1, v2 -; SHADERTEST-NEXT: v_and_b32_e32 v2, 62, v2 -; SHADERTEST-NEXT: v_bfe_u32 v2, s2, v2, 2 -; SHADERTEST-NEXT: v_cmp_eq_u32_e32 vcc, 2, v2 -; SHADERTEST-NEXT: v_cmp_eq_u32_e64 s[0:1], 1, v2 -; SHADERTEST-NEXT: v_cndmask_b32_e32 v4, v3, v1, vcc -; SHADERTEST-NEXT: v_cndmask_b32_e32 v5, v0, v3, vcc -; SHADERTEST-NEXT: v_cndmask_b32_e32 v2, v1, v0, vcc -; SHADERTEST-NEXT: v_cndmask_b32_e64 v0, v4, v0, s[0:1] -; SHADERTEST-NEXT: v_cndmask_b32_e64 v1, v5, v1, s[0:1] -; SHADERTEST-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[0:1] +; SHADERTEST: v_sub_f32_e32 v2, 1.0, v0 +; SHADERTEST-NEXT: s_not_b32 s0, s2 +; SHADERTEST-NEXT: s_and_b32 s1, s2, 0xaaaaaaaa +; SHADERTEST-NEXT: s_lshr_b32 s0, s0, 1 +; SHADERTEST-NEXT: s_and_b32 s2, s2, s0 +; SHADERTEST-NEXT: v_sub_f32_e32 v2, v2, v1 +; SHADERTEST-NEXT: s_bitreplicate_b64_b32 s[0:1], s1 +; SHADERTEST-NEXT: s_and_b32 s2, s2, 0x55555555 +; SHADERTEST-NEXT: s_wqm_b64 vcc, s[0:1] +; SHADERTEST-NEXT: s_bitreplicate_b64_b32 s[0:1], s2 +; SHADERTEST-NEXT: v_cndmask_b32_e32 v3, v2, v1, vcc +; SHADERTEST-NEXT: v_cndmask_b32_e32 v4, v0, v2, vcc +; SHADERTEST-NEXT: v_cndmask_b32_e32 v5, v1, v0, vcc +; SHADERTEST-NEXT: s_wqm_b64 vcc, s[0:1] +; SHADERTEST-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc +; SHADERTEST-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc +; SHADERTEST-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc ; SHADERTEST-NEXT: v_cvt_pkrtz_f16_f32_e32 v0, v0, v1 ; SHADERTEST-NEXT: v_cvt_pkrtz_f16_f32_e64 v1, v2, 1.0 ; SHADERTEST-NEXT: exp mrt0 v0, v0, v1, v1 done compr vm diff --git a/llpc/test/shaderdb/gfx11/SgprUserDataInit_Fs.pipe b/llpc/test/shaderdb/gfx11/SgprUserDataInit_Fs.pipe index f179cb677d..30a256a175 100644 --- a/llpc/test/shaderdb/gfx11/SgprUserDataInit_Fs.pipe +++ b/llpc/test/shaderdb/gfx11/SgprUserDataInit_Fs.pipe @@ -359,7 +359,7 @@ colorBuffer[0].blendSrcAlphaToColor = 0 ; CHECK-NEXT: .user_data_reg_map: ; CHECK-NEXT: - 0x10000000 ; CHECK-NEXT: - 0x12 -; CHECK-NEXT: - 0xffffffff +; CHECK-NEXT: - 0x10000022 ; CHECK-NEXT: - 0xffffffff ; CHECK-NEXT: - 0xffffffff ; CHECK-NEXT: - 0xffffffff diff --git a/llpc/test/shaderdb/multiple_inputs/test_inputs/PipelineVsFs_ConstantData_Vs2Fs1.pipe b/llpc/test/shaderdb/multiple_inputs/test_inputs/PipelineVsFs_ConstantData_Vs2Fs1.pipe index ca9cdc46b3..240c8b3ffc 100644 --- a/llpc/test/shaderdb/multiple_inputs/test_inputs/PipelineVsFs_ConstantData_Vs2Fs1.pipe +++ b/llpc/test/shaderdb/multiple_inputs/test_inputs/PipelineVsFs_ConstantData_Vs2Fs1.pipe @@ -32,8 +32,9 @@ colorBuffer[0].blendEnable = 0 ; SHADERTEST-NEXT: [[TMP3:%.*]] = zext nneg i8 [[TMP2]] to i64 ; SHADERTEST-NEXT: [[TMP4:%.*]] = getelementptr [3 x <4 x float>], ptr addrspace(4) @__llpc_global_proxy_colors, i64 0, i64 [[TMP3]] ; SHADERTEST-NEXT: [[TMP5:%.*]] = load <4 x float>, ptr addrspace(4) [[TMP4]], align 16 -; SHADERTEST-NEXT: [[TMP6:%.*]] = insertvalue { <4 x float> } poison, <4 x float> [[TMP5]], 0 -; SHADERTEST-NEXT: ret { <4 x float> } [[TMP6]] +; SHADERTEST-NEXT: [[TMP6:%.*]] = insertvalue { <4 x float>, i32 } poison, <4 x float> [[TMP5]], 0 +; SHADERTEST-NEXT: [[TMP7:%.*]] = insertvalue { <4 x float>, i32 } [[TMP6]], i32 0, 1 +; SHADERTEST-NEXT: ret { <4 x float>, i32 } [[TMP7]] ; ; ; SHADERTEST2_PP0-LABEL: @_amdgpu_ps_main( diff --git a/llpc/test/shaderdb/object/ObjOutput_TestFsBuiltIn_lit.frag b/llpc/test/shaderdb/object/ObjOutput_TestFsBuiltIn_lit.frag index ace9f05327..e4c0d7c216 100644 --- a/llpc/test/shaderdb/object/ObjOutput_TestFsBuiltIn_lit.frag +++ b/llpc/test/shaderdb/object/ObjOutput_TestFsBuiltIn_lit.frag @@ -19,7 +19,7 @@ void main() ; SHADERTEST: call void @lgc.output.export.builtin.SampleMask.i32.a1i32 ; SHADERTEST: call void @lgc.output.export.builtin.FragStencilRef ; SHADERTEST-LABEL: {{^// LLPC}} pipeline patching results -; SHADERTEST: ret { <4 x float> } +; SHADERTEST: ret { <4 x float>, i32 } ; SHADERTEST: AMDLLPC SUCCESS */ // END_SHADERTEST diff --git a/llpc/test/shaderdb/ray_tracing/PipelineRays_Continufy.pipe b/llpc/test/shaderdb/ray_tracing/PipelineRays_Continufy.pipe new file mode 100644 index 0000000000..586e8ec6bc --- /dev/null +++ b/llpc/test/shaderdb/ray_tracing/PipelineRays_Continufy.pipe @@ -0,0 +1,153 @@ +; Check that the ray tracing continufy mode is working. +; Generating the instruction 'image_bvh64_intersect_ray' indicates the trace ray library is linked correctly. + +; TODO: Change this to ISA / assembly output checks once the LLVM backend has settled + +; RUN: amdllpc -gfxip 11.0 -emit-llvm -o - %s | FileCheck -check-prefixes=CHECK %s + +; CHECK-LABEL: @_amdgpu_cs_main( +; CHECK: call void {{.*}} @llvm.amdgcn.cs.chain. + +; CHECK-LABEL: @_rgen_1( +; CHECK-NOT: br i1 +; CHECK: call void {{.*}} @llvm.amdgcn.cs.chain. + +; CHECK-LABEL: @_rgen_1.resume.0( +; CHECK: call void {{.*}} @llvm.amdgcn.cs.chain. +; CHECK: unreachable +; CHECK: ret void + +; CHECK-LABEL: @_chit_2( +; CHECK-NOT: br i1 +; CHECK: call void {{.*}} @llvm.amdgcn.cs.chain. + +; CHECK-LABEL: @_cs_( +; CHECK: call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray. +; CHECK-NOT: ret void +; CHECK: call void {{.*}} @llvm.amdgcn.cs.chain. +; CHECK-NOT: ret void + +[Version] +version = 69 + +[rgenGlsl] +#version 460 +#extension GL_EXT_ray_tracing : require + +struct RayPayload { + vec3 color; +}; + +layout(binding = 0, set = 0) uniform accelerationStructureEXT g_bvh; +layout(binding = 1, set = 0, rgba32f) uniform image2D g_dst; + +layout(location = 14) rayPayloadEXT RayPayload g_ray; + +void main() { + vec3 origin; + origin.x = gl_LaunchIDEXT.x; + origin.y = gl_LaunchIDEXT.y; + origin.z = 0; + + traceRayEXT(g_bvh, /* ray flags */ 0, /* cull mask */ 0xff, + /* sbt offset */ 0, /* sbt stride */ 1, /* miss index */ 0, + origin.xyz, /* tmin */ 0.0, /* direction */ vec3(1, 0, 0), + /* tmax */ 48.0, /* payload location */ 14); + + imageStore(g_dst, ivec2(gl_LaunchIDEXT.xy), vec4(g_ray.color, 0)); +} + +[rgenInfo] +entryPoint = main + +[chitGlsl] +#version 460 +#extension GL_EXT_ray_tracing : require + +struct RayPayload { + vec3 color; +}; + +layout(shaderRecordEXT, std430) buffer sbt { + float z; +}; + +hitAttributeEXT vec2 g_hit; +rayPayloadInEXT RayPayload g_ray; + +void main() { + g_ray.color.xy = g_hit; + g_ray.color.z = z; +} + +[chitInfo] +entryPoint = main + +[ResourceMapping] +userDataNode[0].visibility = 0xffffffff +userDataNode[0].type = DescriptorTableVaPtr +userDataNode[0].offsetInDwords = 0 +userDataNode[0].sizeInDwords = 1 +userDataNode[0].next[0].type = DescriptorConstBuffer +userDataNode[0].next[0].offsetInDwords = 0 +userDataNode[0].next[0].sizeInDwords = 4 +userDataNode[0].next[0].set = 0x00000000 +userDataNode[0].next[0].binding = 0 +userDataNode[0].next[1].type = DescriptorImage +userDataNode[0].next[1].offsetInDwords = 4 +userDataNode[0].next[1].sizeInDwords = 8 +userDataNode[0].next[1].set = 0x00000000 +userDataNode[0].next[1].binding = 1 +userDataNode[1].visibility = 0xffffffff +userDataNode[1].type = DescriptorTableVaPtr +userDataNode[1].offsetInDwords = 1 +userDataNode[1].sizeInDwords = 1 +userDataNode[1].next[0].type = DescriptorConstBufferCompact +userDataNode[1].next[0].offsetInDwords = 0 +userDataNode[1].next[0].sizeInDwords = 2 +userDataNode[1].next[0].set = 0x0000005D +userDataNode[1].next[0].binding = 17 +userDataNode[1].next[1].type = DescriptorConstBuffer +userDataNode[1].next[1].offsetInDwords = 2 +userDataNode[1].next[1].sizeInDwords = 4 +userDataNode[1].next[1].set = 0x0000005D +userDataNode[1].next[1].binding = 0 +userDataNode[1].next[2].type = DescriptorBuffer +userDataNode[1].next[2].offsetInDwords = 6 +userDataNode[1].next[2].sizeInDwords = 4 +userDataNode[1].next[2].set = 0x0000005D +userDataNode[1].next[2].binding = 1 + +[RayTracingPipelineState] +groups[0].type = VK_RAY_TRACING_SHADER_GROUP_TYPE_GENERAL_KHR +groups[0].generalShader = 0 +groups[0].closestHitShader = -1 +groups[0].anyHitShader = -1 +groups[0].intersectionShader = -1 +groups[1].type = VK_RAY_TRACING_SHADER_GROUP_TYPE_TRIANGLES_HIT_GROUP_KHR +groups[1].closestHitShader = 1 +maxRecursionDepth = 1 +indirectStageMask = 0xffffffff +mode = 2 +rtState.bvhResDescSize = 4 +rtState.bvhResDesc[0] = 0 +rtState.bvhResDesc[1] = 2197815296 +rtState.bvhResDesc[2] = 4294967295 +rtState.bvhResDesc[3] = 2164261887 +rtState.nodeStrideShift = 7 +rtState.threadGroupSizeX = 8 +rtState.threadGroupSizeY = 4 +rtState.threadGroupSizeZ = 1 +rtState.rayQueryCsSwizzle = 1 +rtState.ldsStackSize = 16 +rtState.dispatchRaysThreadGroupSize = 32 +rtState.ldsSizePerThreadGroup = 65536 +rtState.outerTileSize = 4 +rtState.dispatchDimSwizzleMode = 0 +rtState.enableDispatchRaysInnerSwizzle = 1 +rtState.enableDispatchRaysOuterSwizzle = 1 +rtState.enableOptimalLdsStackSizeForIndirect = 1 +rtState.enableOptimalLdsStackSizeForUnified = 1 +payloadSizeMaxInLib = 12 +attributeSizeMaxInLib = 8 +hasPipelineLibrary = 1 diff --git a/llpc/test/shaderdb/relocatable_shaders/PipelineVsFs_EnableColorExport.pipe b/llpc/test/shaderdb/relocatable_shaders/PipelineVsFs_EnableColorExport.pipe index 26b17e7d29..729dace170 100644 --- a/llpc/test/shaderdb/relocatable_shaders/PipelineVsFs_EnableColorExport.pipe +++ b/llpc/test/shaderdb/relocatable_shaders/PipelineVsFs_EnableColorExport.pipe @@ -92,11 +92,10 @@ attribute[0].offset = 0 ; SHADERTEST-NEXT: [[DOTI01:%.*]] = sitofp i32 [[DOTI0]] to float ; SHADERTEST-NEXT: [[DOTI12:%.*]] = sitofp i32 [[DOTI1]] to float ; SHADERTEST-NEXT: [[TMP23:%.*]] = call reassoc nnan nsz arcp contract afn <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float [[DOTI01]], float [[DOTI12]], <8 x i32> [[TMP20]], <4 x i32> [[TMP21]], i1 false, i32 0, i32 0) -; SHADERTEST-NEXT: [[TMP24:%.*]] = insertvalue { <4 x float> } poison, <4 x float> [[TMP23]], 0 ; SHADERTEST-NEXT: [[TMP25:%.*]] = zext i32 [[COLOREXPADDR:%.*]] to i64 ; SHADERTEST-NEXT: [[TMP26:%.*]] = or {{(disjoint )?}}i64 [[TMP12]], [[TMP25]] ; SHADERTEST-NEXT: [[TMP27:%.*]] = inttoptr i64 [[TMP26]] to ptr addrspace(4) -; SHADERTEST-NEXT: call amdgpu_gfx addrspace(4) void [[TMP27]]({ <4 x float> } [[TMP24]]) +; SHADERTEST-NEXT: call amdgpu_gfx addrspace(4) void [[TMP27]](<4 x float> [[TMP23]], i32 inreg 0) #[[ATTR1:[0-9]+]] ; SHADERTEST-NEXT: unreachable ; ; @@ -130,6 +129,7 @@ attribute[0].offset = 0 ; SHADERTEST-NEXT: v_cvt_f32_i32_e32 v1, v2 ; SHADERTEST-NEXT: s_waitcnt lgkmcnt(0) ; SHADERTEST-NEXT: image_sample v[0:3], v[0:1], s[4:11], s[12:15] dmask:0xf dim:SQ_RSRC_IMG_2D +; SHADERTEST-NEXT: s_mov_b32 s4, 0 ; SHADERTEST-NEXT: s_swappc_b64 s[30:31], s[2:3] ; ; diff --git a/llpc/test/shaderdb/relocatable_shaders/PipelineVsFs_MultiDwordPushConst.pipe b/llpc/test/shaderdb/relocatable_shaders/PipelineVsFs_MultiDwordPushConst.pipe index f6b43d8a0f..597d9d0166 100644 --- a/llpc/test/shaderdb/relocatable_shaders/PipelineVsFs_MultiDwordPushConst.pipe +++ b/llpc/test/shaderdb/relocatable_shaders/PipelineVsFs_MultiDwordPushConst.pipe @@ -6,7 +6,7 @@ ; Check that the llvm-ir gets the push constant values from as a parameter. ; SHADERTEST: // LLPC pipeline patching results -; SHADERTEST: define dllexport amdgpu_ps { <4 x float> } @_amdgpu_ps_main({{.*}}, i32 inreg noundef %userdata3, i32 inreg noundef %userdata4, {{.*}}) +; SHADERTEST: define dllexport amdgpu_ps { <4 x float>, i32 } @_amdgpu_ps_main({{.*}}, i32 inreg noundef %userdata3, i32 inreg noundef %userdata4, {{.*}}) ; SHADERTEST: [[pushConst0:%[.a-zA-Z0-9]+]] = bitcast i32 %userdata3 to float ; SHADERTEST: [[pushConst1:%[.a-zA-Z0-9]+]] = bitcast i32 %userdata4 to float ; SHADERTEST: @llvm.amdgcn.image.gather4.lz.2d.sl_v4f32i32s.f32({{.*}}, float [[pushConst0]], float [[pushConst1]], {{.*}}) diff --git a/llpc/translator/include/LLVMSPIRVLib.h b/llpc/translator/include/LLVMSPIRVLib.h index b1170ed7b0..fc85bf8370 100644 --- a/llpc/translator/include/LLVMSPIRVLib.h +++ b/llpc/translator/include/LLVMSPIRVLib.h @@ -53,8 +53,8 @@ class SPIRVModule; /// \brief Represents one entry in specialization constant map. struct SPIRVSpecConstEntry { - uint32_t DataSize; // Data size of specilization constant (in bytes) - const void *Data; // Data of specilization constant + uint32_t DataSize; // Data size of specialization constant (in bytes) + const void *Data; // Data of specialization constant }; /// \brief Represents the map from SpecId to specialization constant data. diff --git a/llpc/translator/lib/SPIRV/SPIRVReader.cpp b/llpc/translator/lib/SPIRV/SPIRVReader.cpp index 100c863725..a19912e44c 100644 --- a/llpc/translator/lib/SPIRV/SPIRVReader.cpp +++ b/llpc/translator/lib/SPIRV/SPIRVReader.cpp @@ -54,6 +54,7 @@ #include "lgc/LgcRtDialect.h" #include "lgc/Pipeline.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Analysis/ConstantFolding.h" #include "llvm/Analysis/VectorUtils.h" @@ -288,21 +289,36 @@ uint64_t SPIRVToLLVM::getTypeStoreSize(Type *const t) { return calculatedSize; } -Value *SPIRVToLLVM::mapValue(SPIRVValue *bv, Value *v) { - auto loc = m_valueMap.find(bv); - if (loc != m_valueMap.end()) { - if (loc->second == v) - return v; - auto ld = dyn_cast(loc->second); - auto placeholder = cast(ld->getPointerOperand()); - assert(ld && placeholder->getName().startswith(KPlaceholderPrefix) && "A value is translated twice"); - // Replaces placeholders for PHI nodes - ld->replaceAllUsesWith(v); - ld->eraseFromParent(); - placeholder->eraseFromParent(); - } - m_valueMap[bv] = v; - return v; +SmallVector SPIRVToLLVM::mapValue(SPIRVValue *bv, ArrayRef values) { + auto oldValues = m_valueMap.lookup(bv); + if (!oldValues.empty()) { + if (oldValues[0] == values[0]) { + assert(llvm::equal(oldValues, values)); + return {}; + } + + // Replace placeholders for PHI nodes + assert(oldValues.size() == values.size()); + for (auto [oldValue, value] : llvm::zip(oldValues, values)) { + auto load = cast(oldValue); + auto placeholder = cast(load->getPointerOperand()); + load->replaceAllUsesWith(value); + load->eraseFromParent(); + placeholder->eraseFromParent(); + } + } + (void)m_valueMap.set(bv, values); + + return SmallVector(values); +} + +// This overload is a convenience that allowed the introduction of multi-value +// translation without too much code churn caused by translation that return +// nullptr values. +SmallVector SPIRVToLLVM::mapValue(SPIRVValue *bv, Value *value) { + if (!value) + return {}; + return mapValue(bv, ArrayRef(value)); } Value *SPIRVToLLVM::mapEntry(const SPIRVEntry *be, Value *v) { @@ -333,11 +349,14 @@ bool SPIRVToLLVM::isSPIRVBuiltinVariable(GlobalVariable *gv, SPIRVBuiltinVariabl return true; } +SmallVector SPIRVToLLVM::getTranslatedValues(SPIRVValue *bv) { + return SmallVector(m_valueMap.lookup(bv)); +} + Value *SPIRVToLLVM::getTranslatedValue(SPIRVValue *bv) { - auto loc = m_valueMap.find(bv); - if (loc != m_valueMap.end()) - return loc->second; - return nullptr; + auto values = getTranslatedValues(bv); + assert(values.size() <= 1); + return values.empty() ? nullptr : values[0]; } void SPIRVToLLVM::setAttrByCalledFunc(CallInst *call) { @@ -1031,19 +1050,28 @@ bool SPIRVToLLVM::isSPIRVCmpInstTransToLLVMInst(SPIRVInstruction *bi) const { return isCmpOpCode(oc); } -void SPIRVToLLVM::setName(llvm::Value *v, SPIRVValue *bv) { +void SPIRVToLLVM::setName(ArrayRef values, SPIRVValue *bv) { const auto &name = bv->getName(); - if (v->hasName()) - return; + for (const auto &value : llvm::enumerate(values)) { + Value *v = value.value(); + if (v->hasName()) + continue; - if (v->getType()->isVoidTy()) - return; + if (v->getType()->isVoidTy()) + continue; - if (!name.empty()) - v->setName(name); - else if (SPIRVInstNamer && isa(v) && bv->hasId()) - v->setName(Twine("spv") + Twine(bv->getId())); + auto suffix = [&]() -> std::string { + if (values.size() == 1) + return {}; + return '.' + std::to_string(value.index()); + }; + + if (!name.empty()) + v->setName(Twine(name) + suffix()); + else if (SPIRVInstNamer && isa(v) && bv->hasId()) + v->setName(Twine("spv") + Twine(bv->getId()) + suffix()); + } } void SPIRVToLLVM::setLLVMLoopMetadata(SPIRVLoopMerge *lm, BranchInst *bi) { @@ -1106,27 +1134,34 @@ void SPIRVToLLVM::setLLVMLoopMetadata(SPIRVLoopMerge *lm, BranchInst *bi) { bi->setMetadata("llvm.loop", node); } -Value *SPIRVToLLVM::transValue(SPIRVValue *bv, Function *f, BasicBlock *bb, bool createPlaceHolder) { - SPIRVToLLVMValueMap::iterator loc = m_valueMap.find(bv); +SmallVector SPIRVToLLVM::transValueMulti(SPIRVValue *bv, Function *f, BasicBlock *bb, bool createPlaceHolder) { + auto values = m_valueMap.lookup(bv); - if (loc != m_valueMap.end() && (!m_placeholderMap.count(bv) || createPlaceHolder)) - return loc->second; + if (!values.empty() && (!m_placeholderMap.count(bv) || createPlaceHolder)) + return SmallVector(values); bv->validate(); auto v = transValueWithoutDecoration(bv, f, bb, createPlaceHolder); - if (!v) - return nullptr; + if (v.empty()) + return {}; + setName(v, bv); if (!transDecoration(bv, v)) { assert(0 && "trans decoration fail"); - return nullptr; + return {}; } m_dbgTran.recordsValue(bv, v); return v; } +Value *SPIRVToLLVM::transValue(SPIRVValue *bv, Function *f, BasicBlock *bb, bool createPlaceHolder) { + auto values = transValueMulti(bv, f, bb, createPlaceHolder); + assert(values.size() <= 1); + return values.empty() ? nullptr : values[0]; +} + Value *SPIRVToLLVM::transConvertInst(SPIRVValue *bv, Function *f, BasicBlock *bb) { SPIRVUnary *bc = static_cast(bv); auto src = transValue(bc->getOperand(0), f, bb, bb != nullptr); @@ -3565,10 +3600,7 @@ template <> Value *SPIRVToLLVM::transValueWithOpcode(SPIRVValue *co // // @param spvValue : A SPIR-V value. template <> Value *SPIRVToLLVM::transValueWithOpcode(SPIRVValue *const spvValue) { - Value *result = nullptr; - - result = getBuilder()->CreateSubgroupElect(); - return result; + return getBuilder()->CreateSubgroupElect(); } // ===================================================================================================================== @@ -3578,16 +3610,11 @@ template <> Value *SPIRVToLLVM::transValueWithOpcode(SPI template <> Value *SPIRVToLLVM::transValueWithOpcode(SPIRVValue *const spvValue) { SPIRVInstruction *const spvInst = static_cast(spvValue); std::vector spvOperands = spvInst->getOperands(); + assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); BasicBlock *const block = getBuilder()->GetInsertBlock(); Function *const func = getBuilder()->GetInsertBlock()->getParent(); Value *const predicate = transValue(spvOperands[1], func, block); - Value *result = nullptr; - - { - assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); - result = getBuilder()->CreateSubgroupAll(predicate); - } - return result; + return getBuilder()->CreateSubgroupAll(predicate); } // ===================================================================================================================== @@ -3597,17 +3624,11 @@ template <> Value *SPIRVToLLVM::transValueWithOpcode(SPIRV template <> Value *SPIRVToLLVM::transValueWithOpcode(SPIRVValue *const spvValue) { SPIRVInstruction *const spvInst = static_cast(spvValue); std::vector spvOperands = spvInst->getOperands(); - + assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); BasicBlock *const block = getBuilder()->GetInsertBlock(); Function *const func = getBuilder()->GetInsertBlock()->getParent(); Value *const predicate = transValue(spvOperands[1], func, block); - Value *result = nullptr; - - { - assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); - result = getBuilder()->CreateSubgroupAny(predicate); - } - return result; + return getBuilder()->CreateSubgroupAny(predicate); } // ===================================================================================================================== @@ -3617,17 +3638,11 @@ template <> Value *SPIRVToLLVM::transValueWithOpcode(SPIRV template <> Value *SPIRVToLLVM::transValueWithOpcode(SPIRVValue *const spvValue) { SPIRVInstruction *const spvInst = static_cast(spvValue); std::vector spvOperands = spvInst->getOperands(); - + assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); BasicBlock *const block = getBuilder()->GetInsertBlock(); Function *const func = getBuilder()->GetInsertBlock()->getParent(); Value *const value = transValue(spvOperands[1], func, block); - Value *result = nullptr; - - { - assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); - result = getBuilder()->CreateSubgroupAllEqual(value); - } - return result; + return getBuilder()->CreateSubgroupAllEqual(value); } // ===================================================================================================================== @@ -3637,18 +3652,13 @@ template <> Value *SPIRVToLLVM::transValueWithOpcode( template <> Value *SPIRVToLLVM::transValueWithOpcode(SPIRVValue *const spvValue) { SPIRVInstruction *const spvInst = static_cast(spvValue); std::vector spvOperands = spvInst->getOperands(); + assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); BasicBlock *const block = getBuilder()->GetInsertBlock(); Function *const func = getBuilder()->GetInsertBlock()->getParent(); Value *const value = transValue(spvOperands[1], func, block); Value *const index = transValue(spvOperands[2], func, block); - Value *result = nullptr; - - { - assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); - result = getBuilder()->CreateSubgroupBroadcastWaterfall(value, index); - } - return result; + return getBuilder()->CreateSubgroupBroadcastWaterfall(value, index); } // ===================================================================================================================== @@ -3658,17 +3668,11 @@ template <> Value *SPIRVToLLVM::transValueWithOpcode template <> Value *SPIRVToLLVM::transValueWithOpcode(SPIRVValue *const spvValue) { SPIRVInstruction *const spvInst = static_cast(spvValue); std::vector spvOperands = spvInst->getOperands(); - + assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); BasicBlock *const block = getBuilder()->GetInsertBlock(); Function *const func = getBuilder()->GetInsertBlock()->getParent(); Value *const value = transValue(spvOperands[1], func, block); - Value *result = nullptr; - - { - assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); - result = getBuilder()->CreateSubgroupBroadcastFirst(value); - } - return result; + return getBuilder()->CreateSubgroupBroadcastFirst(value); } // ===================================================================================================================== @@ -3678,17 +3682,11 @@ template <> Value *SPIRVToLLVM::transValueWithOpcode Value *SPIRVToLLVM::transValueWithOpcode(SPIRVValue *const spvValue) { SPIRVInstruction *const spvInst = static_cast(spvValue); std::vector spvOperands = spvInst->getOperands(); - + assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); BasicBlock *const block = getBuilder()->GetInsertBlock(); Function *const func = getBuilder()->GetInsertBlock()->getParent(); Value *const predicate = transValue(spvOperands[1], func, block); - Value *result = nullptr; - - { - assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); - result = getBuilder()->CreateSubgroupBallot(predicate); - } - return result; + return getBuilder()->CreateSubgroupBallot(predicate); } // ===================================================================================================================== @@ -3698,17 +3696,11 @@ template <> Value *SPIRVToLLVM::transValueWithOpcode(SP template <> Value *SPIRVToLLVM::transValueWithOpcode(SPIRVValue *const spvValue) { SPIRVInstruction *const spvInst = static_cast(spvValue); std::vector spvOperands = spvInst->getOperands(); - + assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); BasicBlock *const block = getBuilder()->GetInsertBlock(); Function *const func = getBuilder()->GetInsertBlock()->getParent(); Value *const value = transValue(spvOperands[1], func, block); - Value *result = nullptr; - - { - assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); - result = getBuilder()->CreateSubgroupInverseBallot(value); - } - return result; + return getBuilder()->CreateSubgroupInverseBallot(value); } // ===================================================================================================================== @@ -3718,18 +3710,12 @@ template <> Value *SPIRVToLLVM::transValueWithOpcode Value *SPIRVToLLVM::transValueWithOpcode(SPIRVValue *const spvValue) { SPIRVInstruction *const spvInst = static_cast(spvValue); std::vector spvOperands = spvInst->getOperands(); - + assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); BasicBlock *const block = getBuilder()->GetInsertBlock(); Function *const func = getBuilder()->GetInsertBlock()->getParent(); Value *const value = transValue(spvOperands[1], func, block); Value *const index = transValue(spvOperands[2], func, block); - Value *result = nullptr; - - { - assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); - result = getBuilder()->CreateSubgroupBallotBitExtract(value, index); - } - return result; + return getBuilder()->CreateSubgroupBallotBitExtract(value, index); } // ===================================================================================================================== @@ -3739,24 +3725,21 @@ template <> Value *SPIRVToLLVM::transValueWithOpcode Value *SPIRVToLLVM::transValueWithOpcode(SPIRVValue *const spvValue) { SPIRVInstruction *const spvInst = static_cast(spvValue); std::vector spvOperands = spvInst->getOperands(); - + assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); BasicBlock *const block = getBuilder()->GetInsertBlock(); Function *const func = getBuilder()->GetInsertBlock()->getParent(); Value *const value = transValue(spvOperands[2], func, block); - { - assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); - switch (static_cast(spvOperands[1])->getZExtIntValue()) { - case GroupOperationReduce: - return getBuilder()->CreateSubgroupBallotBitCount(value); - case GroupOperationInclusiveScan: - return getBuilder()->CreateSubgroupBallotInclusiveBitCount(value); - case GroupOperationExclusiveScan: - return getBuilder()->CreateSubgroupBallotExclusiveBitCount(value); - default: - llvm_unreachable("Should never be called!"); - return nullptr; - } + switch (static_cast(spvOperands[1])->getZExtIntValue()) { + case GroupOperationReduce: + return getBuilder()->CreateSubgroupBallotBitCount(value); + case GroupOperationInclusiveScan: + return getBuilder()->CreateSubgroupBallotInclusiveBitCount(value); + case GroupOperationExclusiveScan: + return getBuilder()->CreateSubgroupBallotExclusiveBitCount(value); + default: + llvm_unreachable("Should never be called!"); + return nullptr; } } @@ -3767,17 +3750,11 @@ template <> Value *SPIRVToLLVM::transValueWithOpcode Value *SPIRVToLLVM::transValueWithOpcode(SPIRVValue *const spvValue) { SPIRVInstruction *const spvInst = static_cast(spvValue); std::vector spvOperands = spvInst->getOperands(); - + assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); BasicBlock *const block = getBuilder()->GetInsertBlock(); Function *const func = getBuilder()->GetInsertBlock()->getParent(); Value *const value = transValue(spvOperands[1], func, block); - Value *result = nullptr; - - { - assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); - result = getBuilder()->CreateSubgroupBallotFindLsb(value); - } - return result; + return getBuilder()->CreateSubgroupBallotFindLsb(value); } // ===================================================================================================================== @@ -3787,17 +3764,11 @@ template <> Value *SPIRVToLLVM::transValueWithOpcode Value *SPIRVToLLVM::transValueWithOpcode(SPIRVValue *const spvValue) { SPIRVInstruction *const spvInst = static_cast(spvValue); std::vector spvOperands = spvInst->getOperands(); - + assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); BasicBlock *const block = getBuilder()->GetInsertBlock(); Function *const func = getBuilder()->GetInsertBlock()->getParent(); Value *const value = transValue(spvOperands[1], func, block); - Value *result = nullptr; - - { - assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); - result = getBuilder()->CreateSubgroupBallotFindMsb(value); - } - return result; + return getBuilder()->CreateSubgroupBallotFindMsb(value); } // ===================================================================================================================== @@ -3807,18 +3778,12 @@ template <> Value *SPIRVToLLVM::transValueWithOpcode Value *SPIRVToLLVM::transValueWithOpcode(SPIRVValue *const spvValue) { SPIRVInstruction *const spvInst = static_cast(spvValue); std::vector spvOperands = spvInst->getOperands(); - + assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); BasicBlock *const block = getBuilder()->GetInsertBlock(); Function *const func = getBuilder()->GetInsertBlock()->getParent(); Value *const value = transValue(spvOperands[1], func, block); Value *const index = transValue(spvOperands[2], func, block); - Value *result = nullptr; - - { - assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); - result = getBuilder()->CreateSubgroupShuffle(value, index); - } - return result; + return getBuilder()->CreateSubgroupShuffle(value, index); } // ===================================================================================================================== @@ -3828,18 +3793,12 @@ template <> Value *SPIRVToLLVM::transValueWithOpcode(S template <> Value *SPIRVToLLVM::transValueWithOpcode(SPIRVValue *const spvValue) { SPIRVInstruction *const spvInst = static_cast(spvValue); std::vector spvOperands = spvInst->getOperands(); - + assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); BasicBlock *const block = getBuilder()->GetInsertBlock(); Function *const func = getBuilder()->GetInsertBlock()->getParent(); Value *const value = transValue(spvOperands[1], func, block); Value *const mask = transValue(spvOperands[2], func, block); - Value *result = nullptr; - - { - assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); - result = getBuilder()->CreateSubgroupShuffleXor(value, mask); - } - return result; + return getBuilder()->CreateSubgroupShuffleXor(value, mask); } // ===================================================================================================================== @@ -3849,18 +3808,12 @@ template <> Value *SPIRVToLLVM::transValueWithOpcode Value *SPIRVToLLVM::transValueWithOpcode(SPIRVValue *const spvValue) { SPIRVInstruction *const spvInst = static_cast(spvValue); std::vector spvOperands = spvInst->getOperands(); - + assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); BasicBlock *const block = getBuilder()->GetInsertBlock(); Function *const func = getBuilder()->GetInsertBlock()->getParent(); Value *const value = transValue(spvOperands[1], func, block); Value *const delta = transValue(spvOperands[2], func, block); - Value *result = nullptr; - - { - assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); - result = getBuilder()->CreateSubgroupShuffleUp(value, delta); - } - return result; + return getBuilder()->CreateSubgroupShuffleUp(value, delta); } // ===================================================================================================================== @@ -3870,18 +3823,12 @@ template <> Value *SPIRVToLLVM::transValueWithOpcode template <> Value *SPIRVToLLVM::transValueWithOpcode(SPIRVValue *const spvValue) { SPIRVInstruction *const spvInst = static_cast(spvValue); std::vector spvOperands = spvInst->getOperands(); - + assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); BasicBlock *const block = getBuilder()->GetInsertBlock(); Function *const func = getBuilder()->GetInsertBlock()->getParent(); Value *const value = transValue(spvOperands[1], func, block); Value *const delta = transValue(spvOperands[2], func, block); - Value *result = nullptr; - - { - assert(static_cast(spvOperands[0])->getZExtIntValue() == ScopeSubgroup); - result = getBuilder()->CreateSubgroupShuffleDown(value, delta); - } - return result; + return getBuilder()->CreateSubgroupShuffleDown(value, delta); } // ===================================================================================================================== @@ -3958,7 +3905,7 @@ template <> Value *SPIRVToLLVM::transValueWithOpcodegetScalarSizeInBits() == 64); gpuAddr = getBuilder()->CreateBitCast(gpuAddr, FixedVectorType::get(getBuilder()->getInt32Ty(), 2)); } - return mapValue(spvValue, gpuAddr); + return gpuAddr; } // ===================================================================================================================== @@ -5195,7 +5142,7 @@ template <> Value *SPIRVToLLVM::transValueWithOpcode(static_cast(spvInst)->getMatrixSatAccumulation()); Value *coopMatrixD = getBuilder()->CreateCooperativeMatrixMulAdd(coopMatrixA, coopMatrixB, coopMatrixC, isSignedA, - isSignedB, isSat, elemBasicTypeC, elemBasicTypeA); + isSignedB, isSat, 0, elemBasicTypeC, elemBasicTypeA); return coopMatrixD; } @@ -5209,7 +5156,8 @@ template <> Value *SPIRVToLLVM::transValueWithOpcode SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *bv, Function *f, BasicBlock *bb, + bool createPlaceHolder) { auto oc = bv->getOpCode(); IntBoolOpMap::rfind(oc, &oc); @@ -5245,7 +5193,7 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *bv, Function *f, Bas } default: llvm_unreachable("Not implemented"); - return nullptr; + return {}; } } @@ -5311,7 +5259,7 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *bv, Function *f, Bas } default: llvm_unreachable("not implemented"); - return nullptr; + return {}; } } @@ -5335,7 +5283,7 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *bv, Function *f, Bas return mapValue(bv, &(*i)); } llvm_unreachable("Invalid argument"); - return nullptr; + return {}; } case OpFunction: @@ -5443,8 +5391,6 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *bv, Function *f, Bas else phiNode = PHINode::Create(transType(phi->getType()), phi->getPairs().size() / 2, phi->getName(), bb); - auto lPhi = dyn_cast(mapValue(bv, phiNode)); - #ifndef NDEBUG SmallDenseSet seenPredecessors; #endif @@ -5452,7 +5398,7 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *bv, Function *f, Bas auto translatedVal = transValue(incomingV, f, bb); auto translatedBb = cast(transValue(incomingBb, f, bb)); - lPhi->addIncoming(translatedVal, translatedBb); + phiNode->addIncoming(translatedVal, translatedBb); #ifndef NDEBUG assert(seenPredecessors.count(translatedBb) == 0 && @@ -5461,7 +5407,7 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *bv, Function *f, Bas #endif }); - return lPhi; + return mapValue(bv, phiNode); } case OpUnreachable: @@ -5502,7 +5448,7 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *bv, Function *f, Bas case OpLine: case OpSelectionMerge: - return nullptr; + return {}; case OpLoopMerge: { // Should be translated at OpBranch or OpBranchConditional cases SPIRVLoopMerge *lm = static_cast(bv); auto continueLabel = m_bm->get(lm->getContinueTarget()); @@ -5955,7 +5901,7 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *bv, Function *f, Bas SPIRVUnary *bc = static_cast(bv); Value *val = transValue(bc->getOperand(0), f, bb); if (!isa(val->getType())) - return val; + return mapValue(bc, val); Value *result = getBuilder()->CreateExtractElement(val, uint64_t(0)); for (unsigned i = 1, e = cast(val->getType())->getNumElements(); i != e; ++i) { Value *elem = getBuilder()->CreateExtractElement(val, i); @@ -6132,7 +6078,7 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *bv, Function *f, Bas return mapValue(bv, getBuilder()->CreateICmpEQ(residentCode, getBuilder()->getInt32(0))); } case OpImageTexelPointer: - return nullptr; + return {}; case OpPtrDiff: { SPIRVBinary *const bi = static_cast(bv); Value *const op1 = @@ -6364,7 +6310,7 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *bv, Function *f, Bas case OpExecuteCallableKHR: return mapValue(bv, transValueWithOpcode(bv)); case OpConvertUToAccelerationStructureKHR: - return transValueWithOpcode(bv); + return mapValue(bv, transValueWithOpcode(bv)); case OpTerminateRayKHR: return mapValue(bv, transValueWithOpcode(bv)); case OpIgnoreIntersectionKHR: @@ -6379,9 +6325,9 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *bv, Function *f, Bas case OpSUDotAccSatKHR: return mapValue(bv, transSPIRVIntegerDotProductFromInst(static_cast(bv), bb)); case OpEmitMeshTasksEXT: - return transValueWithOpcode(bv); + return mapValue(bv, transValueWithOpcode(bv)); case OpSetMeshOutputsEXT: - return transValueWithOpcode(bv); + return mapValue(bv, transValueWithOpcode(bv)); case OpCooperativeMatrixLengthKHR: return mapValue(bv, transValueWithOpcode(bv)); case OpCooperativeMatrixLoadKHR: @@ -6405,7 +6351,7 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *bv, Function *f, Bas } llvm_unreachable("Translation of SPIRV instruction not implemented"); - return NULL; + return {}; } void SPIRVToLLVM::truncConstantIndex(std::vector &indices, BasicBlock *bb) { @@ -6454,8 +6400,8 @@ Function *SPIRVToLLVM::transFunction(SPIRVFunction *bf) { bool isEntry = (entryPoint); SPIRVExecutionModelKind execModel = isEntry ? entryPoint->getExecModel() : ExecutionModelMax; auto linkage = isEntry ? GlobalValue::ExternalLinkage : transLinkageType(bf); - FunctionType *ft = dyn_cast(transType(bf->getFunctionType())); - Function *f = cast(mapValue(bf, Function::Create(ft, linkage, bf->getName(), m_m))); + FunctionType *ft = cast(transType(bf->getFunctionType())); + Function *f = cast(mapValue(bf, Function::Create(ft, linkage, bf->getName(), m_m))[0]); mapFunction(bf, f); if (!f->isIntrinsic()) { if (isEntry) { @@ -8039,12 +7985,6 @@ bool SPIRVToLLVM::transAddressingModel() { return true; } -bool SPIRVToLLVM::transDecoration(SPIRVValue *bv, Value *v) { - if (!transShaderDecoration(bv, v)) - return false; - return true; -} - bool SPIRVToLLVM::transNonTemporalMetadata(Instruction *i) { Constant *one = ConstantInt::get(Type::getInt32Ty(*m_context), 1); MDNode *node = MDNode::get(*m_context, ConstantAsMetadata::get(one)); @@ -8283,6 +8223,12 @@ bool SPIRVToLLVM::transMetadata() { ComputeShaderMode computeMode = {}; + if (bf->getExecutionMode(ExecutionModeDerivativeGroupQuadsNV)) + computeMode.derivatives = DerivativeMode::Quads; + else if (bf->getExecutionMode(ExecutionModeDerivativeGroupLinearNV)) + computeMode.derivatives = DerivativeMode::Linear; + else + computeMode.derivatives = DerivativeMode::None; unsigned overrideShaderGroupSizeX = m_shaderOptions->overrideShaderThreadGroupSizeX; unsigned overrideShaderGroupSizeY = m_shaderOptions->overrideShaderThreadGroupSizeY; unsigned overrideShaderGroupSizeZ = m_shaderOptions->overrideShaderThreadGroupSizeZ; @@ -8339,8 +8285,8 @@ bool SPIRVToLLVM::checkContains64BitType(SPIRVType *bt) { return false; } -bool SPIRVToLLVM::transShaderDecoration(SPIRVValue *bv, Value *v) { - auto gv = dyn_cast(v); +bool SPIRVToLLVM::transDecoration(SPIRVValue *bv, ArrayRef values) { + auto gv = dyn_cast(values[0]); // Some SPIR-V instructions (e.g. OpAccessChain) can become no-ops in LLVM IR, // so we must explicitly check both the SPIR-V opcode and the LLVM type. if (gv && bv->getOpCode() == OpVariable) { @@ -8470,18 +8416,6 @@ bool SPIRVToLLVM::transShaderDecoration(SPIRVValue *bv, Value *v) { inOutDec.XfbOffset = xfbOffset; } - // If dual source blend is dynamically set, need to confirm whether the fragment shader actually uses - // dual-source blending by checking if there is an output at Location 0. If there isn't any output in - // index, the poison value will be exported. - Llpc::Context *llpcContext = static_cast(m_context); - if ((llpcContext->getPipelineType() == PipelineType::Graphics) && (m_execModule == spv::ExecutionModelFragment)) { - auto *buildInfo = static_cast(llpcContext->getPipelineBuildInfo()); - if (buildInfo->cbState.dualSourceBlendDynamic && (as == SPIRAS_Output) && (inOutDec.Value.Loc == 0) && - (inOutDec.Index == 1)) { - llpcContext->getPipelineContext()->setUseDualSourceBlend(true); - } - } - Type *mdTy = nullptr; SPIRVType *bt = bv->getType()->getPointerElementType(); bool vs64BitsAttribInputSingleLoc = (as == SPIRAS_Input && m_execModule == ExecutionModelVertex && @@ -8643,26 +8577,31 @@ bool SPIRVToLLVM::transShaderDecoration(SPIRVValue *bv, Value *v) { } } else { bool isNonUniform = bv->hasDecorate(DecorationNonUniformEXT); - if (isNonUniform && isa(v)) { - std::vector args; - args.push_back(v); - auto types = getTypes(args); - auto voidTy = Type::getVoidTy(*m_context); - auto bb = cast(v)->getParent(); - - // Per-instruction metadata is not safe, LLVM optimizer may remove them, - // so we choose to add a dummy instruction and remove them when it isn't - // needed. - std::string mangledFuncName(gSPIRVMD::NonUniform); - appendTypeMangling(voidTy, args, mangledFuncName); - auto f = getOrCreateFunction(m_m, voidTy, types, mangledFuncName); - if (bb->getTerminator()) { - // NOTE: For OpCopyObject, we use the operand value directly, which may be in another block that already has a - // terminator. In this case, insert the call before the terminator. - assert(bv->getOpCode() == OpCopyObject); - CallInst::Create(f, args, "", bb->getTerminator()); - } else { - CallInst::Create(f, args, "", bb); + if (isNonUniform) { + for (Value *v : values) { + if (!isa(v)) + continue; + + SmallVector args; + args.push_back(v); + auto types = getTypes(args); + auto voidTy = Type::getVoidTy(*m_context); + auto bb = cast(v)->getParent(); + + // Per-instruction metadata is not safe, LLVM optimizer may remove them, + // so we choose to add a dummy instruction and remove them when it isn't + // needed. + std::string mangledFuncName(gSPIRVMD::NonUniform); + appendTypeMangling(voidTy, args, mangledFuncName); + auto f = getOrCreateFunction(m_m, voidTy, types, mangledFuncName); + if (bb->getTerminator()) { + // NOTE: For OpCopyObject, we use the operand value directly, which may be in another block that already has a + // terminator. In this case, insert the call before the terminator. + assert(bv->getOpCode() == OpCopyObject); + CallInst::Create(f, args, "", bb->getTerminator()); + } else { + CallInst::Create(f, args, "", bb); + } } } } @@ -8876,7 +8815,7 @@ Constant *SPIRVToLLVM::buildShaderInOutMetadata(SPIRVType *bt, ShaderInOutDecora if (alignTo64Bit) startXfbExtraOffset = roundUpToMultiple(inOutDec.XfbOffset + inOutDec.XfbExtraOffset, 8u) - inOutDec.XfbOffset; - // PerVertex is not inherted. + // PerVertex is not inherited. bool isPerVertexDimension = inOutDec.PerVertexDimension; inOutDec.PerVertexDimension = false; @@ -10249,20 +10188,10 @@ void SPIRVToLLVM::insertScratchBoundsChecks(SPIRVValue *memOp, const ScratchBoun getBuilder()->SetInsertPoint(&finalBlock->front()); PHINode *convergePhi = getBuilder()->CreatePHI(lastMemOp->getType(), 2); lastMemOp->replaceAllUsesWith(convergePhi); + m_valueMap.replaceAllUsesOfWith(lastMemOp, convergePhi); convergePhi->addIncoming(Constant::getNullValue(lastMemOp->getType()), checkBlock); convergePhi->addIncoming(lastMemOp, thenTerminator->getParent()); - - if (scratchBoundsCheckData.instructionOrigin) { - auto it = m_valueMap.find(scratchBoundsCheckData.instructionOrigin); - if (it != m_valueMap.end()) - m_valueMap[scratchBoundsCheckData.instructionOrigin] = convergePhi; - } - - for (auto &it : m_valueMap) { - if (it.second == lastMemOp) - it.second = convergePhi; - } } // Re-create all instructions before the terminator in the then block. @@ -10348,8 +10277,7 @@ void SPIRVToLLVM::createXfbMetadata(bool hasXfbOuts) { // Update indexOfBuffer for block array, the N array-elements are captured by N consecutive buffers. SPIRVType *bt = bv->getType()->getPointerElementType(); if (bt->isTypeArray()) { - assert(m_valueMap.find(bv) != m_valueMap.end()); - auto output = cast(m_valueMap[bv]); + auto output = cast(getTranslatedValue(bv)); MDNode *metaNode = output->getMetadata(gSPIRVMD::InOut); assert(metaNode); auto elemMeta = mdconst::dyn_extract(metaNode->getOperand(0)); diff --git a/llpc/translator/lib/SPIRV/SPIRVReader.h b/llpc/translator/lib/SPIRV/SPIRVReader.h index 7a6da16136..a9be457be3 100644 --- a/llpc/translator/lib/SPIRV/SPIRVReader.h +++ b/llpc/translator/lib/SPIRV/SPIRVReader.h @@ -43,6 +43,7 @@ #include "SPIRVInternal.h" #include "SPIRVModule.h" #include "SPIRVToLLVMDbgTran.h" +#include "compilerutils/LoweringPointerTupleMap.h" #include "vkgcDefs.h" #include "lgc/Builder.h" @@ -91,8 +92,10 @@ class SPIRVToLLVM { bool translate(ExecutionModel entryExecModel, const char *entryName); bool transAddressingModel(); + SmallVector transValueMulti(SPIRVValue *, Function *f, BasicBlock *bb, bool createPlaceHolder = true); Value *transValue(SPIRVValue *, Function *f, BasicBlock *, bool createPlaceHolder = true); - Value *transValueWithoutDecoration(SPIRVValue *, Function *f, BasicBlock *, bool createPlaceHolder = true); + SmallVector transValueWithoutDecoration(SPIRVValue *, Function *f, BasicBlock *, + bool createPlaceHolder = true); Value *transAtomicRMW(SPIRVValue *, const AtomicRMWInst::BinOp); Constant *transInitializer(SPIRVValue *, Type *); template Value *transValueWithOpcode(SPIRVValue *); @@ -105,8 +108,7 @@ class SPIRVToLLVM { Value *indexDescPtr(Type *elementTy, Value *base, Value *index); Value *transGroupArithOp(lgc::Builder::GroupArithOp, SPIRVValue *); - bool transDecoration(SPIRVValue *, Value *); - bool transShaderDecoration(SPIRVValue *, Value *); + bool transDecoration(SPIRVValue *, ArrayRef); bool checkContains64BitType(SPIRVType *bt); Constant *buildShaderInOutMetadata(SPIRVType *bt, ShaderInOutDecorate &inOutDec, Type *&metaTy, bool vs64bitsAttrib); Constant *buildShaderBlockMetadata(SPIRVType *bt, ShaderBlockDecorate &blockDec, Type *&mdTy, @@ -199,6 +201,7 @@ class SPIRVToLLVM { // Post-process translated LLVM module to undo row major matrices. bool postProcessRowMajorMatrix(); + SmallVector getTranslatedValues(SPIRVValue *bv); Value *getTranslatedValue(SPIRVValue *bv); // Create !lgc.xfb.state metadata @@ -225,7 +228,7 @@ class SPIRVToLLVM { typedef DenseMap SPIRVToLLVMFullTypeMap; typedef DenseMap SPIRVToLLVMTypeMap; - typedef DenseMap SPIRVToLLVMValueMap; + typedef compilerutils::LoweringPointerTupleMap SPIRVToLLVMValueMap; typedef DenseMap SPIRVBlockToLLVMStructMap; typedef DenseMap SPIRVToLLVMFunctionMap; typedef DenseMap BuiltinVarMap; @@ -359,7 +362,8 @@ class SPIRVToLLVM { // If a value is mapped twice, the existing mapped value is a placeholder, // which must be a load instruction of a global variable whose name starts // with kPlaceholderPrefix. - Value *mapValue(SPIRVValue *bv, Value *v); + SmallVector mapValue(SPIRVValue *bv, ArrayRef values); + SmallVector mapValue(SPIRVValue *bv, Value *values); // Keep track of the spirvEntry Value *mapEntry(const SPIRVEntry *be, Value *v); @@ -400,7 +404,7 @@ class SPIRVToLLVM { llvm::Value *transShiftLogicalBitwiseInst(SPIRVValue *bv, BasicBlock *bb, Function *f); llvm::Value *transCmpInst(SPIRVValue *bv, BasicBlock *bb, Function *f); - void setName(llvm::Value *v, SPIRVValue *bv); + void setName(ArrayRef values, SPIRVValue *bv); void setLLVMLoopMetadata(SPIRVLoopMerge *lm, BranchInst *bi); template bool foreachFuncCtlMask(Source, Func); llvm::GlobalValue::LinkageTypes transLinkageType(const SPIRVValue *v); diff --git a/llpc/translator/lib/SPIRV/SPIRVToLLVMDbgTran.cpp b/llpc/translator/lib/SPIRV/SPIRVToLLVMDbgTran.cpp index 985ab99fdc..97a5b6673c 100644 --- a/llpc/translator/lib/SPIRV/SPIRVToLLVMDbgTran.cpp +++ b/llpc/translator/lib/SPIRV/SPIRVToLLVMDbgTran.cpp @@ -111,17 +111,19 @@ unsigned SPIRVToLLVMDbgTran::getConstant(const SPIRVId id) { // ===================================================================================================================== // Record spirv/llvm variables for later debug info processing -void SPIRVToLLVMDbgTran::recordsValue(SPIRVValue *SV, Value *V) { - if (!Enable || !SV->hasLine()) +void SPIRVToLLVMDbgTran::recordsValue(SPIRVValue *sv, ArrayRef values) { + if (!Enable || !sv->hasLine()) return; // A constant sampler does not have a corresponding SPIRVInstruction. - if (SV->getOpCode() == OpConstantSampler) + if (sv->getOpCode() == OpConstantSampler) return; - if (Instruction *I = dyn_cast(V)) { - SPIRVInstruction *SI = static_cast(SV); - if (RecordedInstructions.find(I) == RecordedInstructions.end()) { - RecordedInstructions[I] = SI; + for (Value *value : values) { + if (Instruction *inst = dyn_cast(value)) { + SPIRVInstruction *si = static_cast(sv); + if (RecordedInstructions.find(inst) == RecordedInstructions.end()) { + RecordedInstructions[inst] = si; + } } } } diff --git a/llpc/translator/lib/SPIRV/SPIRVToLLVMDbgTran.h b/llpc/translator/lib/SPIRV/SPIRVToLLVMDbgTran.h index a75a82a1ae..44a50f214f 100644 --- a/llpc/translator/lib/SPIRV/SPIRVToLLVMDbgTran.h +++ b/llpc/translator/lib/SPIRV/SPIRVToLLVMDbgTran.h @@ -65,7 +65,7 @@ class SPIRVToLLVMDbgTran { SPIRVToLLVMDbgTran(SPIRVModule *TBM, Module *TM, SPIRVToLLVM *Reader); void createCompilationUnit(); - void recordsValue(SPIRVValue *SV, Value *V); + void recordsValue(SPIRVValue *SV, ArrayRef V); void applyDelayedDbgInfo(); template T *transDebugInst(const SPIRVExtInst *DebugInst) { assert((DebugInst->getExtSetKind() == SPIRVEIS_Debug || diff --git a/llpc/translator/lib/SPIRV/libSPIRV/SPIRVEnum.h b/llpc/translator/lib/SPIRV/libSPIRV/SPIRVEnum.h index 53cdddf9f5..3fb4f407e8 100644 --- a/llpc/translator/lib/SPIRV/libSPIRV/SPIRVEnum.h +++ b/llpc/translator/lib/SPIRV/libSPIRV/SPIRVEnum.h @@ -211,6 +211,8 @@ template <> inline void SPIRVMap::init() { ADD_VEC_INIT(CapabilityMeshShadingEXT, {CapabilityShader}); ADD_VEC_INIT(CapabilityFragmentBarycentricKHR, {CapabilityShader}); ADD_VEC_INIT(CapabilityCooperativeMatrixKHR, {CapabilityShader}); + ADD_VEC_INIT(CapabilityComputeDerivativeGroupLinearNV, {CapabilityShader}); + ADD_VEC_INIT(CapabilityComputeDerivativeGroupQuadsNV, {CapabilityShader}); } template <> inline void SPIRVMap::init() { diff --git a/llpc/translator/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h b/llpc/translator/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h index ba830ab07d..4439a4a4db 100644 --- a/llpc/translator/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h +++ b/llpc/translator/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h @@ -560,6 +560,8 @@ inline bool isValid(spv::Capability V) { case CapabilityWorkgroupMemoryExplicitLayoutKHR: case CapabilityWorkgroupMemoryExplicitLayout8BitAccessKHR: case CapabilityWorkgroupMemoryExplicitLayout16BitAccessKHR: + case CapabilityComputeDerivativeGroupLinearNV: + case CapabilityComputeDerivativeGroupQuadsNV: return true; default: return false; diff --git a/llpc/translator/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h b/llpc/translator/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h index 90836b1f17..9b17bde3c0 100644 --- a/llpc/translator/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h +++ b/llpc/translator/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h @@ -500,6 +500,8 @@ template <> inline void SPIRVMap::init() { add(CapabilityWorkgroupMemoryExplicitLayout8BitAccessKHR, "WorkgroupMemoryExplicitLayout8BitAccessKHR"); add(CapabilityWorkgroupMemoryExplicitLayout16BitAccessKHR, "WorkgroupMemoryExplicitLayout16BitAccessKHR"); add(CapabilityCooperativeMatrixKHR, "CooperativeMatrixKHR"); + add(CapabilityComputeDerivativeGroupLinearNV, "ComputeDerivativeGroupLinearNV"); + add(CapabilityComputeDerivativeGroupQuadsNV, "ComputeDerivativeGroupQuadsNV"); } SPIRV_DEF_NAMEMAP(Capability, SPIRVCapabilityNameMap) diff --git a/llpc/translator/lib/SPIRV/libSPIRV/SPIRVType.cpp b/llpc/translator/lib/SPIRV/libSPIRV/SPIRVType.cpp index 757c754f58..4669efb22b 100644 --- a/llpc/translator/lib/SPIRV/libSPIRV/SPIRVType.cpp +++ b/llpc/translator/lib/SPIRV/libSPIRV/SPIRVType.cpp @@ -64,7 +64,7 @@ SPIRVWord SPIRVType::getBitWidth() const { if (isTypeMatrix()) return getMatrixColumnType()->getBitWidth(); if (isTypeBool()) - return 1; + return 32; return isTypeInt() ? getIntegerBitWidth() : getFloatBitWidth(); } @@ -76,7 +76,7 @@ SPIRVWord SPIRVType::getFloatBitWidth() const { SPIRVWord SPIRVType::getIntegerBitWidth() const { assert((OpCode == OpTypeInt || OpCode == OpTypeBool) && "Not an integer type"); if (isTypeBool()) - return 1; + return 32; return static_cast(this)->getBitWidth(); } diff --git a/llpc/util/llpcElfWriter.cpp b/llpc/util/llpcElfWriter.cpp index 7a7f5ded83..8681fdc1ba 100644 --- a/llpc/util/llpcElfWriter.cpp +++ b/llpc/util/llpcElfWriter.cpp @@ -341,6 +341,7 @@ void ElfWriter::mergeMetaNote(Context *pContext, const ElfNote *pNote1, con PalAbi::GraphicsRegisterMetadataKey::DbShaderControl, PalAbi::GraphicsRegisterMetadataKey::CbShaderMask, PalAbi::GraphicsRegisterMetadataKey::AaCoverageToShaderSelect, + PalAbi::GraphicsRegisterMetadataKey::PsLoadProvokingVtx, }; auto destRegisters = destPipeline.getMap(true)[PalAbi::PipelineMetadataKey::GraphicsRegisters].getMap(true); auto srcRegisters = srcPipeline.getMap(true)[PalAbi::PipelineMetadataKey::GraphicsRegisters].getMap(true); @@ -351,7 +352,7 @@ void ElfWriter::mergeMetaNote(Context *pContext, const ElfNote *pNote1, con std::string destBlob; destDocument.writeToBlob(destBlob); *pNewNote = *pNote1; - auto data = new uint8_t[destBlob.size() + 4]; // 4 is for additional alignment spece + auto data = new uint8_t[destBlob.size() + 4]; // 4 is for additional alignment space memcpy(data, destBlob.data(), destBlob.size()); pNewNote->hdr.descSize = destBlob.size(); pNewNote->data = data; diff --git a/shared/README.md b/shared/README.md index 7afd8a804c..fa8962b633 100644 --- a/shared/README.md +++ b/shared/README.md @@ -1,7 +1 @@ -# Shared compiler code - -This repository contains code that is shared between different graphics compilers and drivers. - -- [Continuations](./continuations) contains a collection of passes to convert shaders to coroutines. -- [CompilerUtils](./compilerutils) is a library of helpers for frontendy functionality shared by different compiler stacks. -- [LgcRt](./lgcrt) contains the lgc.rt dialect definition and helpers. +Note: The contents of this directory will eventually be moved to the top level of the repository. diff --git a/shared/continuations/CMakeLists.txt b/shared/continuations/CMakeLists.txt index 6c97f64589..26eff9d596 100644 --- a/shared/continuations/CMakeLists.txt +++ b/shared/continuations/CMakeLists.txt @@ -22,7 +22,6 @@ add_llvm_library(LLVMContinuations lib/DXILContIntrinsicPrepare.cpp lib/DXILContLgcRtOpConverter.cpp lib/DXILContPostProcess.cpp - lib/PreCoroutineLowering.cpp lib/DXILMetadata.cpp lib/DXILSupport.cpp lib/GpurtDialect.cpp @@ -58,7 +57,7 @@ target_include_directories(LLVMContinuations PUBLIC llvm_map_components_to_libnames(extra_llvm_libs CompilerUtils) -target_link_libraries(LLVMContinuations PUBLIC ${extra_llvm_libs}) +target_link_libraries(LLVMContinuations PUBLIC llvm_dialects ${extra_llvm_libs} llpc_version) set_compiler_options(LLVMContinuations) # TableGen for dialects diff --git a/shared/continuations/include/continuations/Continuations.h b/shared/continuations/include/continuations/Continuations.h index e61defe6c7..875dcf0dc9 100644 --- a/shared/continuations/include/continuations/Continuations.h +++ b/shared/continuations/include/continuations/Continuations.h @@ -73,6 +73,7 @@ #ifndef CONTINUATIONS_CONTINUATIONS_H #define CONTINUATIONS_CONTINUATIONS_H +#include "compilerutils/CompilerUtils.h" #include "continuations/ContinuationsUtil.h" #include "continuations/PayloadAccessQualifiers.h" #include "llvm-dialects/Dialect/Builder.h" @@ -117,12 +118,9 @@ std::pair moveContinuationStackOffset(IRBuilder<> &B, /// Convert an offset to the continuation stack to a pointer into the memory /// where the continuation stack lives. -Value *continuationStackOffsetToPtr(IRBuilder<> &B, Value *Offset); - -/// Create a new function based on another function, copying attributes and -/// other properties. -Function *cloneFunctionHeader(Function &F, FunctionType *NewType, - ArrayRef ArgAttrs); +Value *continuationStackOffsetToPtr(IRBuilder<> &B, Value *Offset, + Module &GpurtLibrary, + CompilerUtils::CrossModuleInliner &Inliner); /// Create a new function, as cloneFunctionHeader, but include types metadata. Function *cloneFunctionHeaderWithTypes(Function &F, DXILContFuncTy &NewType, @@ -150,10 +148,6 @@ Function *getSetLocalRootIndex(Module &M); /// Get intrinsic to convert a dx handle to an acceleration struct address. Function *getAccelStructAddr(Module &M, Type *HandleTy); -/// Get intrinsic to save the continuation state. -Function *getContinuationSaveContinuationState(Module &M); -/// Get intrinsic to restore the continuation state. -Function *getContinuationRestoreContinuationState(Module &M); /// Get the continuation.continue intrinsic. Function *getContinuationContinue(Module &M); /// Get the continuation.waitContinue intrinsic. @@ -186,13 +180,6 @@ uint64_t getInlineHitAttrsBytes(Module &M); /// Extract a function from a constant metadata node, ignoring any bitcasts. Function *extractFunctionOrNull(Metadata *N); -/// Returns true if a call to the given function should be rematerialized -/// in a shader of the specified kind. -/// -/// If no shader kind is specified, -bool isRematerializableLgcRtOp( - CallInst &CInst, std::optional Kind = std::nullopt); - /// Recurse into the first member of the given SystemData to find an object of /// the wanted type. /// See also the system data documentation at the top of Continuations.h. @@ -203,7 +190,8 @@ Value *getDXILSystemData(IRBuilder<> &B, Value *SystemData, Type *SystemDataTy, /// implementation (_cont_*). CallInst *replaceIntrinsicCall(IRBuilder<> &B, Type *SystemDataTy, Value *SystemData, DXILShaderKind Kind, - CallInst *Call); + CallInst *Call, Module *GpurtLibrary, + CompilerUtils::CrossModuleInliner &Inliner); /// Transformations that run early on the driver/gpurt module. /// @@ -264,51 +252,15 @@ class DialectContextAnalysis class LegacyCleanupContinuationsPass : public llvm::PassInfoMixin { public: - LegacyCleanupContinuationsPass(); + LegacyCleanupContinuationsPass(llvm::Module *GpurtLibrary = nullptr) + : GpurtLibrary(GpurtLibrary) {} llvm::PreservedAnalyses run(llvm::Module &Module, llvm::ModuleAnalysisManager &AnalysisManager); static llvm::StringRef name() { return "legacy continuation cleanup"; } private: - struct ContinuationData { - /// All functions belonging to this continuation, the entry function is the - /// first one - SmallVector Functions; - /// Size of the continuation state in byte - uint32_t ContStateBytes = 0; - CallInst *MallocCall = nullptr; - MDNode *MD = nullptr; - AllocaInst *NewContState = nullptr; - SmallVector NewFunctions; - SmallVector NewReturnContinues; - /// Cleaned entry function, used to replace metadata - Function *NewStart = nullptr; - }; - - void analyzeContinuation(Function &F, MDNode *MD); - void processContinuations(); - void handleFunctionEntry(IRBuilder<> &B, ContinuationData &Data, Function *F, - bool IsEntry); - void handleContinue(IRBuilder<> &B, ContinuationData &Data, Instruction *Ret); - void handleSingleContinue(IRBuilder<> &B, ContinuationData &Data, - CallInst *Call, Value *ResumeFun); - void handleReturn(IRBuilder<> &B, ContinuationData &Data, CallInst *ContRet); - - Module *M; - Type *I32; - Type *I64; - Function *ContMalloc; - Function *ContFree; - Function *SaveContState; - Function *RestoreContState; - Function *RegisterBufferSetPointerBarrier; - Function *Continue; - Function *WaitContinue; - Function *Complete; - GlobalVariable *ContState; - MapVector ToProcess; - uint32_t MaxContStateBytes; + Module *GpurtLibrary; }; class CleanupContinuationsPass @@ -354,11 +306,15 @@ class CleanupContinuationsPass class LowerRaytracingPipelinePass : public llvm::PassInfoMixin { public: - LowerRaytracingPipelinePass() = default; + LowerRaytracingPipelinePass(llvm::Module *GpurtLibrary = nullptr) + : GpurtLibrary(GpurtLibrary) {} llvm::PreservedAnalyses run(llvm::Module &Module, llvm::ModuleAnalysisManager &AnalysisManager); static llvm::StringRef name() { return "Lower raytracing pipeline pass"; } + +private: + Module *GpurtLibrary; }; class DXILContIntrinsicPreparePass @@ -373,73 +329,18 @@ class DXILContIntrinsicPreparePass } }; -class PreCoroutineLoweringPass - : public llvm::PassInfoMixin { -public: - PreCoroutineLoweringPass(); - llvm::PreservedAnalyses run(llvm::Module &Module, - llvm::ModuleAnalysisManager &AnalysisManager); - - static llvm::StringRef name() { - return "Continuation pre coroutine preparation"; - } - -private: - bool splitBB(); - bool removeInlinedIntrinsics(); - bool lowerGetShaderKind(); - bool lowerGetCurrentFuncAddr(); - - Module *Mod; -}; - class DXILContPostProcessPass : public llvm::PassInfoMixin { public: - DXILContPostProcessPass(); + DXILContPostProcessPass(llvm::Module *GpurtLibrary = nullptr) + : GpurtLibrary(GpurtLibrary) {} llvm::PreservedAnalyses run(llvm::Module &Module, llvm::ModuleAnalysisManager &AnalysisManager); static llvm::StringRef name() { return "DXIL continuation post processing"; } - struct FunctionData { - DXILShaderKind Kind = DXILShaderKind::Invalid; - /// Calls to hlsl intrinsics - SmallVector IntrinsicCalls; - /// Calls to get the system data pointer - SmallVector GetSystemDataCalls; - - /// If this is the start function part of a split function - bool IsStart = true; - /// Pointer to the alloca'd system data object in this function - Value *SystemData = nullptr; - Type *SystemDataTy = nullptr; - }; - private: - // Returns whether changes were made - bool - lowerGetResumePointAddr(llvm::Module &M, llvm::IRBuilder<> &B, - const MapVector &ToProcess); - void handleInitialContinuationStackPtr(IRBuilder<> &B, Function &F); - void handleLgcRtIntrinsic(Function &F); - void handleRegisterBufferSetPointerBarrier(Function &F, - GlobalVariable *Payload); - void handleRegisterBufferGetPointer(IRBuilder<> &B, Function &F, - GlobalVariable *Payload); - void handleValueI32Count(IRBuilder<> &B, Function &F); - void handleValueGetI32(IRBuilder<> &B, Function &F); - void handleValueSetI32(IRBuilder<> &B, Function &F); - - void handleContPayloadRegisterI32Count(Function &F); - void handleContPayloadRegistersGetI32(IRBuilder<> &B, Function &F); - void handleContPayloadRegistersSetI32(IRBuilder<> &B, Function &F); - void handleContStackAlloc(FunctionAnalysisManager &FAM, IRBuilder<> &B, - Function &F); - - Module *Mod; - GlobalVariable *Registers; - MapVector ToProcess; + Module *GpurtLibrary; }; class LowerAwaitPass : public llvm::PassInfoMixin { @@ -486,23 +387,18 @@ class RegisterBufferPass : public llvm::PassInfoMixin { class SaveContinuationStatePass : public llvm::PassInfoMixin { public: - SaveContinuationStatePass(); + SaveContinuationStatePass() = default; llvm::PreservedAnalyses run(llvm::Module &Module, llvm::ModuleAnalysisManager &AnalysisManager); static llvm::StringRef name() { return "save continuation state"; } private: - /// Returns true if something changed. - bool lowerCalls(Function *Intr, bool IsSave); - bool lowerContStateGetPointer(); - void lowerCsp(Function *GetCsp); Type *I32; IRBuilder<> *B; Module *Mod; - GlobalVariable *ContState; }; // No-op pass running before the DXIL continuations pipeline, e.g. for usage diff --git a/shared/continuations/include/continuations/ContinuationsUtil.h b/shared/continuations/include/continuations/ContinuationsUtil.h index eecc624cad..c4d1362a1b 100644 --- a/shared/continuations/include/continuations/ContinuationsUtil.h +++ b/shared/continuations/include/continuations/ContinuationsUtil.h @@ -39,6 +39,7 @@ #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" @@ -70,10 +71,6 @@ class Type; const unsigned RegisterBytes = 4; /// Address space used for globals that should be put into registers. const unsigned GlobalRegisterAddrspace = 20; -/// Amount of registers reserved for the continuation state. -/// Spill everything into memory. No explicit memory address needed, which is -/// instead derived from the CSP. -const unsigned ContinuationStateRegisterCount = 0; /// The (first) register used for the memory pointer in payload registers. /// Currently, it is only a single register for the 32-bit pointer. const unsigned FirstPayloadMemoryPointerRegister = 0; @@ -314,11 +311,10 @@ class DXILContHelper { static constexpr const char *MDTypesFunctionName = "function"; static constexpr const char *MDTypesVoidName = "void"; static constexpr const char *MDDXILPayloadTyName = "dxil.payload.type"; - static constexpr const char *MDLgcCpsModule = "lgc.cps.module"; + static constexpr const char *MDLgcCpsModuleName = "lgc.cps.module"; // Global variable names static constexpr const char *GlobalPayloadName = "PAYLOAD"; - static constexpr const char *GlobalContStateName = "CONTINUATION_STATE"; static constexpr const char *GlobalRegistersName = "REGISTERS"; static constexpr ContStackAddrspace DefaultStackAddrspace = ContStackAddrspace::Scratch; @@ -326,11 +322,19 @@ class DXILContHelper { static void RegisterPasses(llvm::PassBuilder &PB, bool NeedDialectContext); // Registers the generic Continuation pipeline to a LLVM Module Pass manager. - static void addContinuationPasses(llvm::ModulePassManager &MPM); + static void addContinuationPasses(llvm::ModulePassManager &MPM, + llvm::Module *GpurtLibrary); // Registers the DXIL-specific Continuation pipeline to a LLVM Module Pass // manager. - static void addDxilContinuationPasses(llvm::ModulePassManager &MPM); + static void addDxilContinuationPasses(llvm::ModulePassManager &MPM, + llvm::Module *GpurtLibrary = nullptr); + + // Registers the DXIL-specific pipeline for the driver library module to a + // LLVM Module Pass manager. These passes preprocess the driver library into a + // form that can be used for the later continuation passes that are run on app + // modules. + static void addDxilGpurtLibraryPasses(llvm::ModulePassManager &MPM); // Set metadata specifying the number of outgoing payload registers. static void setOutgoingRegisterCount(Instruction *I, uint32_t RegisterCount) { @@ -412,11 +416,8 @@ class DXILContHelper { auto *Registers = M.getGlobalVariable(GlobalRegistersName); if (!Registers) return {}; - const uint32_t NumRegistersI32s = - Registers->getValueType()->getArrayNumElements(); - assert(NumRegistersI32s >= ContinuationStateRegisterCount); const uint32_t NumPayloadRegistersI32s = - NumRegistersI32s - ContinuationStateRegisterCount; + Registers->getValueType()->getArrayNumElements(); assert(NumPayloadRegistersI32s >= tryGetMinPayloadRegisterCount(M).value_or(NumPayloadRegistersI32s)); assert(NumPayloadRegistersI32s <= @@ -521,11 +522,6 @@ class DXILContHelper { return extractZExtI32Constant(F.getMetadata(MDStateName)); } - static bool isTraversal(Function &F) { - // TODO: Make this more robust somehow, restricting to library functions. - return F.getName().contains("Traversal"); - } - static Type *getPayloadTypeFromMetadata(const Function &Func) { if (MDNode *Node = Func.getMetadata(MDDXILPayloadTyName)) return getPayloadTypeFromMetadata(Node); @@ -544,7 +540,7 @@ class DXILContHelper { } static bool isLgcCpsModule(Module &Mod) { - return Mod.getNamedMetadata(MDLgcCpsModule) != nullptr; + return Mod.getNamedMetadata(MDLgcCpsModuleName) != nullptr; } // Specifies that an awaited call should wait on a wait mask. @@ -582,7 +578,7 @@ class DXILContHelper { } } - static lgc::rt::RayTracingShaderStage + static std::optional dxilShaderKindToShaderStage(DXILShaderKind Kind) { switch (Kind) { case DXILShaderKind::RayGeneration: @@ -598,11 +594,17 @@ class DXILContHelper { case DXILShaderKind::Callable: return lgc::rt::RayTracingShaderStage::Callable; default: - report_fatal_error(Twine("Cannot convert DXILShaderKind ") + - Twine(static_cast(Kind)) + - " to RayTracingShaderStage"); + return std::nullopt; } } + + /// Returns true if a call to the given function should be rematerialized + /// in a shader of the specified kind. + /// + /// If no shader kind is specified, return false. + static bool + isRematerializableLgcRtOp(CallInst &CInst, + std::optional Kind = std::nullopt); }; /// Free-standing helpers. @@ -638,6 +640,26 @@ findIntrImplEntryByIntrinsicCall(CallInst *Call); // at the end of LowerRaytracingPipeline. bool removeUnusedFunctionDecls(Module *Mod, bool OnlyIntrinsics = true); +// For each basic block in Func, find the terminator. If it is contained in +// TerminatorOpcodes, then apply the callback on the terminator. +template >> +void forEachTerminator(Function *Func, ArrayRef TerminatorOpcodes, + CallbackTy Callback) { + for (auto &BB : *Func) { + auto *Terminator = BB.getTerminator(); + if (llvm::find(TerminatorOpcodes, Terminator->getOpcode()) != + TerminatorOpcodes.end()) + Callback(*Terminator); + } +} + +// Essentially RAUW for pointers for the case that these use different address +// spaces, rewriting all derived pointers to also use the new address space. +void replaceAllPointerUses(IRBuilder<> *Builder, Value *OldPointerValue, + Value *NewPointerValue, + SmallVectorImpl &ToBeRemoved); + // Replacement for PointerType::getWithSamePointeeType that works with new LLVM. // Returns a typed pointer type if the pointer type is typed. PointerType *getWithSamePointeeType(PointerType *PtrTy, unsigned AddressSpace); diff --git a/shared/continuations/include/continuations/LowerRaytracingPipeline.h b/shared/continuations/include/continuations/LowerRaytracingPipeline.h deleted file mode 100644 index 4ea976fbd5..0000000000 --- a/shared/continuations/include/continuations/LowerRaytracingPipeline.h +++ /dev/null @@ -1,324 +0,0 @@ -/* - *********************************************************************************************************************** - * - * Copyright (c) 2022-2023 Advanced Micro Devices, Inc. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - *deal in the Software without restriction, including without limitation the - *rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - *sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - *all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - *FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - *IN THE SOFTWARE. - * - **********************************************************************************************************************/ - -//===- LowerRaytracingPipeline.h ------------------------------------------===// -// -// Declare the class which implements the lowering of lgc.rt operations within -// the coroutine support. -// -//===----------------------------------------------------------------------===// - -#ifndef CONTINUATIONS_LOWERRAYTRACINGPIPELINE_H -#define CONTINUATIONS_LOWERRAYTRACINGPIPELINE_H - -#include "continuations/ContinuationsUtil.h" -#include "continuations/PayloadAccessQualifiers.h" -#include "lgc/LgcCpsDialect.h" -#include "llvm-dialects/Dialect/Builder.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/IR/IRBuilder.h" -#include - -namespace llvm { -class AllocaInst; -class CallInst; -class Function; -class GlobalVariable; -class Instruction; -class Module; -class StructType; -class Type; -class Value; - -namespace { -enum class ContinuationCallType { - Traversal, - CallShader, - AnyHit, -}; -} // namespace - -class ModuleMetadataState final { -public: - ModuleMetadataState(llvm::Module &Module); - ModuleMetadataState(const ModuleMetadataState &) = delete; - ModuleMetadataState(ModuleMetadataState &&) = default; - - uint32_t getMaxPayloadRegisterCount() const { - return MaxPayloadRegisterCount; - } - - uint32_t getMinPayloadRegisterCount() const { - return MinPayloadRegisterCount; - } - - ContStackAddrspace getContStackAddrspace() const { return StackAddrspace; } - - void updateModuleMetadata() const; - -private: - Module &Mod; - /// MaxPayloadRegisterCount is initialized from metadata. If there is none, - /// use this default instead: - static constexpr uint32_t DefaultPayloadRegisterCount = 30; - /// Maximum allowed number of registers to be used for the payload. - uint32_t MaxPayloadRegisterCount = 0; - /// Minimum required number of payload registers. - uint32_t MinPayloadRegisterCount = 0; - /// The address space used for the continuations stack. - /// Either stack or global memory. - ContStackAddrspace StackAddrspace = DXILContHelper::DefaultStackAddrspace; -}; - -class CpsMutator final { -public: - explicit CpsMutator(Module &Mod) - : Mod{Mod}, IsModuleInCpsMode{DXILContHelper::isLgcCpsModule(Mod)}, - Builder{std::make_unique(Mod.getContext())} {} - - Value *insertCpsAwait(Type *ReturnTy, Value *ShaderAddr, Instruction *Call, - ArrayRef Args, ContinuationCallType CallType, - lgc::cps::CpsShaderStage ShaderStage); - - bool shouldRun() const { return IsModuleInCpsMode; } - -private: - Module &Mod; - bool IsModuleInCpsMode = false; - std::unique_ptr Builder; -}; - -class LowerRaytracingPipelinePassImpl final { -public: - LowerRaytracingPipelinePassImpl(Module &M); - bool run(); - -private: - struct FunctionConfig { - // Maximum allowed size of hit attributes to be used in a TraceRay together - // with this function, even if this function does not touch hit attributes - // (e.g. a Miss shader). - uint32_t MaxHitAttributeBytes = 0; - - bool operator==(const FunctionConfig &Other) const { - return MaxHitAttributeBytes == Other.MaxHitAttributeBytes; - } - std::string getFunctionSuffix() const { - std::string Result; - raw_string_ostream Stream{Result}; - Stream << ".attr_max_" << MaxHitAttributeBytes << "_bytes"; - return Result; - } - }; - - struct FunctionData { - DXILShaderKind Kind = DXILShaderKind::Invalid; - SmallVector TraceRayCalls; - SmallVector ReportHitCalls; - SmallVector CallShaderCalls; - /// Calls to hlsl intrinsics that cannot be rematerialized - SmallVector IntrinsicCalls; - SmallVector ShaderIndexCalls; - - /// Pointer to the alloca'd system data object in this function - AllocaInst *SystemData = nullptr; - StructType *SystemDataTy = nullptr; - Type *ReturnTy = nullptr; - /// Maximum number of I32s required to store the outgoing payload in all - /// CallShader or TraceRay (maximum over all TraceRay formats) calls - uint32_t MaxOutgoingPayloadI32s = 0; - /// Type of the incoming payload - Type *IncomingPayload = nullptr; - FunctionConfig FuncConfig = {}; - /// Serialization info for the incoming payload, if there is one. - /// Also applies to the outgoing payload in that case. - PAQSerializationInfoBase *IncomingPayloadSerializationInfo = nullptr; - /// hit attributes type, incoming for AnyHit and ClosestHit, outgoing for - /// Intersection - Type *HitAttributes = nullptr; - }; - - /// Needed data for handling the end of a function - struct FunctionEndData { - Instruction *Terminator = nullptr; - const PAQSerializationLayout *OutgoingSerializationLayout = nullptr; - SmallVector SavedRegisterValues; - Value *NewPayload = nullptr; - std::optional ShaderStage; - Value *HitAttrsAlloca = nullptr; - Value *OrigHitAttrsAlloca = nullptr; - Type *NewRetTy = nullptr; - }; - - struct AwaitFunctionData { - DXILShaderKind CallerKind = DXILShaderKind::Invalid; - ContinuationCallType CallType; - SmallVector AwaitCalls; - FunctionConfig FuncConfig = {}; - - bool operator==(const AwaitFunctionData &Other) const { - return std::tie(CallType, AwaitCalls, FuncConfig) == - std::tie(Other.CallType, Other.AwaitCalls, FuncConfig); - } - }; - - static DXILShaderKind callTypeToShaderKind(ContinuationCallType CallType); - - CallInst *replaceCall(IRBuilder<> &B, FunctionData &Data, CallInst *Call, - Function *Func, StringRef NewName, - ContinuationCallType CallType); - void handleRestoreSystemData(IRBuilder<> &B, CallInst *Call); - void replaceTraceRayCall(IRBuilder<> &B, FunctionData &Data, CallInst *Call); - void replaceCallShaderCall(IRBuilder<> &B, FunctionData &Data, - CallInst *Call); - void replaceContinuationCall(IRBuilder<> &B, ContinuationCallType CallType, - CallInst *Call, const FunctionConfig &FuncConfig, - DXILShaderKind CallerKind); - void replaceReportHitCall(llvm_dialects::Builder &B, FunctionData &Data, - CallInst *Call); - - void handleReportHit(FunctionData &Data, Function &F); - - void replaceShaderIndexCall(IRBuilder<> &B, FunctionData &Data, - CallInst *Call); - - void handleGetFuncAddr(Function &Func); - - void handleAmdInternalFunc(Function &Func); - - void handleUnrematerializableCandidates(); - - void collectDriverFunctions(); - - // Copy the payload content between global payload and local payload. - // Excludes the stack pointer or hit attributes which may also reside in - // payload storage. If Stage is not set, all fields in SerializationInfo are - // copied. Used for CallShader accesses which are not PAQ qualified and do not - // have PAQShaderStage values. - // If CopiedNodes is set, nodes contained will not be copied, and all copied - // nodes are added to it. - void copyPayload(IRBuilder<> &B, Type &PayloadTy, Value *LocalPayload, - std::optional Stage, - PAQAccessKind GlobalAccessKind, - const PAQSerializationLayout &Layout, - SmallDenseSet *CopiedNodes = nullptr); - - // Special handling for case of copying the result payload of a traceray call - // back to the local payload of the caller. - // This is needed to implement the ClosestHitOut/MissOut optimization. - // We first perform a copy using the ClosestHitOut layout, and then perform an - // additional copy using the MissOut layout, skipping any fields already - // copied (i.e. only copying write(miss) : read(caller) fields). - void copyTraceRayPayloadIncomingToCaller( - IRBuilder<> &B, const PAQTraceRaySerializationInfo &PAQSerializationInfo, - Value *LocalPayload); - - // Caller-save payload registers before CallShader() or TraceRay(), - // which can override payload registers. A register needs to be saved - // if it is live in OutgoingLayout, and not written in OutgoingLayout. - // This includes the payload memory pointer if present. - // SavedRegisters maps indices of payload registers to their saved values. - void savePayloadRegistersBeforeRecursion( - IRBuilder<> &B, DXILShaderKind Kind, - const PAQSerializationLayout &IncomingLayout, - const PAQSerializationLayout &OutgoingLayout, - SmallVectorImpl &SavedRegisterValues); - - // Restore previously saved registers. - void restorePayloadRegistersAfterRecursion( - IRBuilder<> &B, const SmallVectorImpl &SavedRegisterValues); - - void createPayloadGlobal(); - - void setTraversalRegisterCountMetadata(); - - void copyHitAttributes(IRBuilder<> &B, FunctionData &Data, Value *SystemData, - Type *SystemDataTy, Value *LocalHitAttributes, - bool GlobalToLocal, - const PAQSerializationLayout *Layout); - void processContinuations(); - void processFunctionEntry(llvm_dialects::Builder &B, Function *F, - FunctionData &Data); - void processFunctionEnd(llvm_dialects::Builder &B, FunctionData &Data, - FunctionEndData &EData); - void processFunction(llvm_dialects::Builder &B, Function *F, - FunctionData &FuncData); - - void collectProcessableFunctions(); - - void handleDriverFuncAssertions(); - - constexpr static uint32_t ArgContState = 0; - constexpr static uint32_t ArgReturnAddr = 1; - constexpr static uint32_t ArgShaderIndex = 2; - constexpr static uint32_t ArgSystemData = 3; - constexpr static uint32_t ArgHitAttributes = 4; - - MapVector ToProcess; - MapVector AwaitsToProcess; - Module *Mod; - LLVMContext *Context; - const DataLayout *DL; - ModuleMetadataState MetadataState; - CpsMutator Mutator; - PAQSerializationInfoManager PAQManager; - Type *I32; - Type *TokenTy; - /// System data type passed to Traversal - Type *TraversalDataTy; - /// System data type passed to ClosestHit and Miss - Type *HitMissDataTy; - GlobalVariable *Payload; - - // Function definitions and declarations from HLSL - // Driver implementation that returns if AcceptHitAndEndSearch was called - Function *IsEndSearch; - // Driver implementations to set and get the triangle hit attributes from - // system data - Function *GetTriangleHitAttributes; - Function *SetTriangleHitAttributes; - Function *GetLocalRootIndex; - Function *SetLocalRootIndex; - Function *SetupRayGen; - Function *TraceRay; - Function *CallShader; - Function *ReportHit; - Function *AcceptHit; - - Function *RegisterBufferSetPointerBarrier; - - SmallVector Awaits; - SmallVector RestoreSystemDatas; - SmallVector EntriesWithPayloadTypeMetadata; - - // We specialize certain intrinsics that lead to suspend-points (TraceRay, - // CallShader, ReportHit) based on the payload or hit attribute type. - // We store these types (either payload or hit attribute) here for later use. - DenseMap PayloadOrAttrTypesForSpecializedFunctions; -}; - -} // namespace llvm - -#endif diff --git a/shared/continuations/include/continuations/PayloadAccessQualifiers.h b/shared/continuations/include/continuations/PayloadAccessQualifiers.h index 6b1402e133..7b6dda331f 100644 --- a/shared/continuations/include/continuations/PayloadAccessQualifiers.h +++ b/shared/continuations/include/continuations/PayloadAccessQualifiers.h @@ -878,7 +878,8 @@ struct PAQCallShaderSerializationInfo : public PAQSerializationInfoBase { // and caching already seen serialization infos. class PAQSerializationInfoManager { public: - PAQSerializationInfoManager(Module *M, uint32_t MaxPayloadRegisterCount); + PAQSerializationInfoManager(Module *M, Module *GpurtLibrary, + uint32_t MaxPayloadRegisterCount); PAQSerializationInfoManager(const PAQSerializationInfoManager &) = delete; PAQSerializationInfoManager(PAQSerializationInfoManager &&) = default; @@ -950,6 +951,7 @@ class PAQSerializationInfoManager { private: Module *Mod = {}; + Module *GpurtLibrary = {}; uint32_t MaxPayloadRegisterCount = {}; // Stores per-payload-type data diff --git a/shared/continuations/lib/CleanupContinuations.cpp b/shared/continuations/lib/CleanupContinuations.cpp index c46a0fab20..d5e2f67590 100644 --- a/shared/continuations/lib/CleanupContinuations.cpp +++ b/shared/continuations/lib/CleanupContinuations.cpp @@ -56,6 +56,7 @@ // 4. Allocating/freeing cps stack space as needed. //===----------------------------------------------------------------------===// +#include "compilerutils/CompilerUtils.h" #include "continuations/Continuations.h" #include "continuations/ContinuationsDialect.h" #include "lgc/LgcCpsDialect.h" @@ -200,70 +201,9 @@ void CleanupContinuationsPass::updateCpsStack(Function *F, Function *NewFunc, SmallVector ToBeRemoved; Value *OldBase = getContinuationFramePtr(F, IsStart, CpsInfo, ToBeRemoved); - OldBase->mutateType(Builder->getPtrTy(cps::stackAddrSpace)); - - // Traversal through the users and setup the addrspace for the cps stack - // pointers. - SmallVector Worklist(OldBase->users()); - OldBase->replaceAllUsesWith(CpsStack); - - while (!Worklist.empty()) { - Value *Ptr = Worklist.pop_back_val(); - Instruction *Inst = cast(Ptr); - LLVM_DEBUG(dbgs() << "Visiting " << *Inst << '\n'); - switch (Inst->getOpcode()) { - default: - LLVM_DEBUG(Inst->dump()); - llvm_unreachable("Unhandled instruction\n"); - break; - case Instruction::Call: { - if (Inst->isLifetimeStartOrEnd()) { - // The lifetime marker is not useful anymore. - Inst->eraseFromParent(); - } else { - LLVM_DEBUG(Inst->dump()); - llvm_unreachable("Unhandled call instruction\n"); - } - // No further processing needed for the users. - continue; - } - case Instruction::Load: - case Instruction::Store: - // No further processing needed for the users. - continue; - case Instruction::And: - case Instruction::Add: - case Instruction::PtrToInt: - break; - case Instruction::AddrSpaceCast: - assert(Inst->getOperand(0)->getType()->getPointerAddressSpace() == - cps::stackAddrSpace); - // Push the correct users before RAUW. - Worklist.append(Ptr->users().begin(), Ptr->users().end()); - Inst->mutateType(Builder->getPtrTy(cps::stackAddrSpace)); - Inst->replaceAllUsesWith(Inst->getOperand(0)); - ToBeRemoved.push_back(Inst); - continue; - case Instruction::IntToPtr: - case Instruction::GetElementPtr: { - Inst->mutateType(Builder->getPtrTy(cps::stackAddrSpace)); - break; - } - case Instruction::Select: { - // check whether the result type is already what we want - auto *OldType = Inst->getType(); - auto *NewType = Builder->getPtrTy(cps::stackAddrSpace); - if (OldType != NewType) { - Inst->mutateType(NewType); - break; - } - // No further processing if the type is not changed. - continue; - } - } - Worklist.append(Ptr->users().begin(), Ptr->users().end()); - } + replaceAllPointerUses(Builder, OldBase, CpsStack, ToBeRemoved); + for (auto *I : reverse(ToBeRemoved)) I->eraseFromParent(); } @@ -432,7 +372,8 @@ void CleanupContinuationsPass::processContinuations() { auto &Context = F->getContext(); auto *NewFuncTy = FunctionType::get(Type::getVoidTy(Context), AllArgTypes, false); - Function *NewFunc = cloneFunctionHeader(*F, NewFuncTy, ParamAttrs); + Function *NewFunc = + CompilerUtils::cloneFunctionHeader(*F, NewFuncTy, ParamAttrs); NewFunc->takeName(F); FuncData.second.NewFunctions.push_back(NewFunc); diff --git a/shared/continuations/lib/ContinuationsUtil.cpp b/shared/continuations/lib/ContinuationsUtil.cpp index bda3dc83ff..48999e82f2 100644 --- a/shared/continuations/lib/ContinuationsUtil.cpp +++ b/shared/continuations/lib/ContinuationsUtil.cpp @@ -32,10 +32,13 @@ #include "continuations/ContinuationsUtil.h" #include "lgc/LgcRtDialect.h" +#include "llvm-dialects/Dialect/OpSet.h" #include "llvm/ADT/STLExtras.h" #include "llvm/IR/Function.h" #include "llvm/IR/Instructions.h" +#define DEBUG_TYPE "continuations-util" + #define GPURTMAP_ENTRY(Op, GpurtName, AccessesHitData) \ { \ OpDescription::get(), { GpurtName, AccessesHitData } \ @@ -111,6 +114,137 @@ bool llvm::removeUnusedFunctionDecls(Module *Mod, bool OnlyIntrinsics) { return DidChange; } +bool DXILContHelper::isRematerializableLgcRtOp( + CallInst &CInst, std::optional Kind) { + using namespace lgc::rt; + Function *Callee = CInst.getCalledFunction(); + if (!DialectUtils::isLgcRtOp(Callee)) + return false; + + // Always rematerialize + static const OpSet RematerializableDialectOps = + OpSet::get(); + if (RematerializableDialectOps.contains(*Callee)) + return true; + + // Rematerialize for Intersection that can only call ReportHit, which keeps + // the largest system data struct. These cannot be rematerialized in + // ClosestHit, because if ClosestHit calls TraceRay or CallShader, that + // information is lost from the system data struct. Also exclude rayTCurrent + // because ReportHit calls can change that. + if (!Kind || *Kind == DXILShaderKind::Intersection) { + static const OpSet RematerializableIntersectionDialectOps = + OpSet::get(); + if (RematerializableIntersectionDialectOps.contains(*Callee)) + return true; + } + + return false; +} + +void llvm::replaceAllPointerUses(IRBuilder<> *Builder, Value *OldPointerValue, + Value *NewPointerValue, + SmallVectorImpl &ToBeRemoved) { + // Note: The implementation explicitly supports typed pointers, which + // complicates some of the code below. + + // Assert that both types are pointers that only differ in the address space. + PointerType *OldPtrTy = cast(OldPointerValue->getType()); + PointerType *NewPtrTy = cast(NewPointerValue->getType()); + unsigned NewAS = NewPtrTy->getAddressSpace(); + assert(NewAS != OldPtrTy->getAddressSpace()); + assert(PointerType::getWithSamePointeeType(OldPtrTy, NewAS) == NewPtrTy); + + OldPointerValue->mutateType(NewPtrTy); + + // Traverse through the users and setup the addrspace + SmallVector Worklist(OldPointerValue->users()); + OldPointerValue->replaceAllUsesWith(NewPointerValue); + + // Given a pointer type, get a pointer with the same pointee type (possibly + // opaque) as the given type that uses the NewAS address space. + auto GetMutatedPtrTy = [NewAS](Type *Ty) { + PointerType *PtrTy = cast(Ty); + // Support typed pointers: + return getWithSamePointeeType(PtrTy, NewAS); + }; + + while (!Worklist.empty()) { + Value *Ptr = Worklist.pop_back_val(); + Instruction *Inst = cast(Ptr); + LLVM_DEBUG(dbgs() << "Visiting " << *Inst << '\n'); + // In the switch below, "break" means to continue with replacing + // the users of the current value, while "continue" means to stop at + // the current value, and proceed with next one from the work list. + switch (Inst->getOpcode()) { + default: + LLVM_DEBUG(Inst->dump()); + llvm_unreachable("Unhandled instruction\n"); + break; + case Instruction::Call: { + if (Inst->isLifetimeStartOrEnd()) { + // The lifetime marker is not useful anymore. + Inst->eraseFromParent(); + } else { + LLVM_DEBUG(Inst->dump()); + llvm_unreachable("Unhandled call instruction\n"); + } + // No further processing needed for the users. + continue; + } + case Instruction::Load: + case Instruction::Store: + // No further processing needed for the users. + continue; + case Instruction::And: + case Instruction::Add: + case Instruction::PtrToInt: + break; + case Instruction::BitCast: { + // This can happen with typed pointers + auto *BC = cast(Inst); + assert(cast(Inst)->getSrcTy()->isPointerTy() && + BC->getDestTy()->isPointerTy()); + Inst->mutateType(GetMutatedPtrTy(Inst->getType())); + break; + } + case Instruction::AddrSpaceCast: + // Check that the pointer operand has already been fixed + assert(Inst->getOperand(0)->getType()->getPointerAddressSpace() == NewAS); + // Push the correct users before RAUW. + Worklist.append(Ptr->users().begin(), Ptr->users().end()); + Inst->mutateType(GetMutatedPtrTy(Inst->getType())); + // Since we are mutating the address spaces of users as well, + // we can just use the (already mutated) cast operand. + Inst->replaceAllUsesWith(Inst->getOperand(0)); + ToBeRemoved.push_back(Inst); + continue; + case Instruction::IntToPtr: + case Instruction::GetElementPtr: { + Inst->mutateType(GetMutatedPtrTy(Inst->getType())); + break; + } + case Instruction::Select: { + auto *OldType = Inst->getType(); + if (OldType->isPointerTy()) { + Type *NewType = GetMutatedPtrTy(OldType); + // No further processing if the type has the correct pointer type + if (NewType == OldType) + continue; + + Inst->mutateType(NewType); + } + break; + } + } + + Worklist.append(Ptr->users().begin(), Ptr->users().end()); + } +} + PointerType *llvm::getWithSamePointeeType(PointerType *PtrTy, unsigned AddressSpace) { #if LLVM_MAIN_REVISION && LLVM_MAIN_REVISION < 482880 diff --git a/shared/continuations/lib/CpsStackLowering.cpp b/shared/continuations/lib/CpsStackLowering.cpp index 615815492a..5cfddfc4ec 100644 --- a/shared/continuations/lib/CpsStackLowering.cpp +++ b/shared/continuations/lib/CpsStackLowering.cpp @@ -173,10 +173,8 @@ void CpsStackLowering::visitCpsAlloc(lgc::cps::AllocOp &AllocOp) { IRBuilder<> Builder(&AllocOp); Value *Size = AllocOp.getSize(); - const DataLayout &Layout = Mod->getDataLayout(); - Value *VSP = Builder.CreateAlignedLoad( - Builder.getPtrTy(LoweredCpsStackAddrSpace), CpsStackAlloca, - Align(getLoweredCpsStackPointerSize(Layout))); + Value *VSP = Builder.CreateLoad(Builder.getPtrTy(LoweredCpsStackAddrSpace), + CpsStackAlloca); unsigned AlignedSize = alignTo(cast(Size)->getZExtValue(), ContinuationStackAlignment); StackSizeInBytes += AlignedSize; @@ -184,8 +182,7 @@ void CpsStackLowering::visitCpsAlloc(lgc::cps::AllocOp &AllocOp) { // update stack pointer Value *Ptr = Builder.CreateConstGEP1_32(Builder.getInt8Ty(), VSP, AlignedSize); - Builder.CreateAlignedStore(Ptr, CpsStackAlloca, - Align(getLoweredCpsStackPointerSize(Layout))); + Builder.CreateStore(Ptr, CpsStackAlloca); TypeLower.replaceInstruction(&AllocOp, {VSP}); } @@ -196,19 +193,16 @@ void CpsStackLowering::visitCpsAlloc(lgc::cps::AllocOp &AllocOp) { // @param function : the instruction void CpsStackLowering::visitCpsFree(lgc::cps::FreeOp &FreeOp) { IRBuilder<> Builder(&FreeOp); - const DataLayout &Layout = Mod->getDataLayout(); - Value *VSP = Builder.CreateAlignedLoad( - Builder.getPtrTy(LoweredCpsStackAddrSpace), CpsStackAlloca, - Align(getLoweredCpsStackPointerSize(Layout))); + Value *VSP = Builder.CreateLoad(Builder.getPtrTy(LoweredCpsStackAddrSpace), + CpsStackAlloca); Value *Size = FreeOp.getSize(); unsigned AlignedSize = alignTo(cast(Size)->getZExtValue(), ContinuationStackAlignment); Value *Ptr = Builder.CreateConstGEP1_32(Builder.getInt8Ty(), VSP, -AlignedSize); // Assuming continuation stack grows upward. - Builder.CreateAlignedStore(Ptr, CpsStackAlloca, - Align(getLoweredCpsStackPointerSize(Layout))); + Builder.CreateStore(Ptr, CpsStackAlloca); TypeLower.replaceInstruction(&FreeOp, {}); } @@ -218,11 +212,9 @@ void CpsStackLowering::visitCpsFree(lgc::cps::FreeOp &FreeOp) { // @param function : the instruction void CpsStackLowering::visitCpsPeek(lgc::cps::PeekOp &PeekOp) { IRBuilder<> Builder(&PeekOp); - const DataLayout &Layout = Mod->getDataLayout(); - auto *Ptr = Builder.CreateAlignedLoad( - Builder.getPtrTy(LoweredCpsStackAddrSpace), CpsStackAlloca, - Align(getLoweredCpsStackPointerSize(Layout))); + auto *Ptr = Builder.CreateLoad(Builder.getPtrTy(LoweredCpsStackAddrSpace), + CpsStackAlloca); auto *Size = PeekOp.getSize(); unsigned ImmSize = cast(Size)->getZExtValue(); ImmSize = alignTo(ImmSize, ContinuationStackAlignment); @@ -238,12 +230,10 @@ void CpsStackLowering::visitCpsPeek(lgc::cps::PeekOp &PeekOp) { // @param function : the instruction void CpsStackLowering::visitSetVsp(lgc::cps::SetVspOp &SetVsp) { IRBuilder<> B(&SetVsp); - const DataLayout &Layout = Mod->getDataLayout(); auto *Ptr = SetVsp.getPtr(); auto Converted = TypeLower.getValue(Ptr); - B.CreateAlignedStore(Converted[0], CpsStackAlloca, - Align(getLoweredCpsStackPointerSize(Layout))); + B.CreateStore(Converted[0], CpsStackAlloca); TypeLower.replaceInstruction(&SetVsp, {}); } @@ -253,10 +243,8 @@ void CpsStackLowering::visitSetVsp(lgc::cps::SetVspOp &SetVsp) { // @param function : the instruction void CpsStackLowering::visitGetVsp(lgc::cps::GetVspOp &GetVsp) { IRBuilder<> B(&GetVsp); - const DataLayout &Layout = Mod->getDataLayout(); auto *Ptr = - B.CreateAlignedLoad(B.getPtrTy(LoweredCpsStackAddrSpace), CpsStackAlloca, - Align(getLoweredCpsStackPointerSize(Layout))); + B.CreateLoad(B.getPtrTy(LoweredCpsStackAddrSpace), CpsStackAlloca); TypeLower.replaceInstruction(&GetVsp, {Ptr}); } diff --git a/shared/continuations/lib/DXILCont.cpp b/shared/continuations/lib/DXILCont.cpp index 46da4202de..077e6be5b1 100644 --- a/shared/continuations/lib/DXILCont.cpp +++ b/shared/continuations/lib/DXILCont.cpp @@ -29,14 +29,15 @@ // //===----------------------------------------------------------------------===// +#include "compilerutils/CompilerUtils.h" #include "continuations/Continuations.h" #include "continuations/ContinuationsDialect.h" #include "continuations/ContinuationsUtil.h" -#include "continuations/LowerRaytracingPipeline.h" #include "lgc/LgcCpsDialect.h" #include "lgc/LgcRtDialect.h" #include "llvm-dialects/Dialect/Builder.h" #include "llvm-dialects/Dialect/Dialect.h" +#include "llvm/IR/PassManager.h" #include "llvm/Passes/PassBuilder.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Transforms/Coroutines/CoroCleanup.h" @@ -44,6 +45,7 @@ #include "llvm/Transforms/Coroutines/CoroElide.h" #include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/Scalar/ADCE.h" +#include "llvm/Transforms/Scalar/InstSimplifyPass.h" #include "llvm/Transforms/Scalar/SROA.h" #include "llvm/Transforms/Scalar/SimplifyCFG.h" #include "llvm/Transforms/Utils/FixIrreducible.h" @@ -187,15 +189,12 @@ void DXILContHelper::RegisterPasses(PassBuilder &PB, bool NeedDialectContext) { } } -void DXILContHelper::addContinuationPasses(ModulePassManager &MPM) { - MPM.addPass(LowerRaytracingPipelinePass()); - - // Inline TraceRay and similar intrinsic implementations +void DXILContHelper::addContinuationPasses(ModulePassManager &MPM, + Module *GpurtLibrary) { + // Inline functions into shaders, so everything is in a shader MPM.addPass(AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/false)); - // Splits basic blocks after the systemDataRestored marker and removes already - // inlined intrinsic implementations - MPM.addPass(PreCoroutineLoweringPass()); + MPM.addPass(LowerRaytracingPipelinePass(GpurtLibrary)); // Convert the system data struct to a value, so it isn't stored in the // continuation state @@ -204,16 +203,14 @@ void DXILContHelper::addContinuationPasses(ModulePassManager &MPM) { MPM.addPass(LowerAwaitPass()); MPM.addPass(CoroEarlyPass()); - CGSCCPassManager CGPM; - CGPM.addPass(DXILCoroSplitPass()); - MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM))); + MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(DXILCoroSplitPass())); MPM.addPass(createModuleToFunctionPassAdaptor(CoroElidePass())); MPM.addPass(CoroCleanupPass()); - MPM.addPass(LegacyCleanupContinuationsPass()); + MPM.addPass(LegacyCleanupContinuationsPass(GpurtLibrary)); MPM.addPass(RegisterBufferPass()); MPM.addPass(SaveContinuationStatePass()); - MPM.addPass(DXILContPostProcessPass()); + MPM.addPass(DXILContPostProcessPass(GpurtLibrary)); MPM.addPass(RemoveTypesMetadataPass()); @@ -225,19 +222,17 @@ void DXILContHelper::addContinuationPasses(ModulePassManager &MPM) { // pass can still change the module in its preprocessing, lowering switches to // chained ifs. MPM.addPass(createModuleToFunctionPassAdaptor(FixIrreduciblePass())); - - // Inline remaining intrinsic implementations - MPM.addPass(AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/false)); } -void DXILContHelper::addDxilContinuationPasses(ModulePassManager &MPM) { +void DXILContHelper::addDxilContinuationPasses(ModulePassManager &MPM, + Module *GpurtLibrary) { MPM.addPass(DXILContPreHookPass()); // Translate dx.op intrinsic calls to lgc.rt dialect intrinsic calls MPM.addPass(DXILContLgcRtOpConverterPass()); // Add the generic continuations pipeline - addContinuationPasses(MPM); + addContinuationPasses(MPM, GpurtLibrary); // Remove dead instructions using the continuation token, which the translator // can't translate @@ -249,6 +244,20 @@ void DXILContHelper::addDxilContinuationPasses(ModulePassManager &MPM) { MPM.addPass(DXILContPostHookPass()); } +void DXILContHelper::addDxilGpurtLibraryPasses(ModulePassManager &MPM) { + MPM.addPass(llvm::DXILContIntrinsicPreparePass()); + MPM.addPass(AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/false)); + + // Run some light optimizations to remove code guarded by intrinsics that were + // replaced in the prepare pass. + FunctionPassManager FPM; + FPM.addPass(SROAPass(SROAOptions::ModifyCFG)); + FPM.addPass(InstSimplifyPass()); + FPM.addPass(SimplifyCFGPass()); + FPM.addPass(ADCEPass()); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); +} + AnalysisKey DialectContextAnalysis::Key; DialectContextAnalysis::DialectContextAnalysis(bool NeedDialectContext) @@ -267,18 +276,6 @@ DialectContextAnalysis::run(llvm::Module &M, return DialectContextAnalysis::Result(); } -llvm::PreservedAnalyses -LowerRaytracingPipelinePass::run(llvm::Module &M, - llvm::ModuleAnalysisManager &AnalysisManager) { - LLVM_DEBUG(dbgs() << "Run the pass lower-raytracing-pipeline\n"); - AnalysisManager.getResult(M); - - LowerRaytracingPipelinePassImpl Impl(M); - bool Changed = Impl.run(); - - return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); -} - std::pair llvm::moveContinuationStackOffset(IRBuilder<> &B, int32_t I) { // %cont.frame.mem = load i32, i32* %csp @@ -295,7 +292,10 @@ std::pair llvm::moveContinuationStackOffset(IRBuilder<> &B, return std::make_pair(OldCsp, NewCsp); } -Value *llvm::continuationStackOffsetToPtr(IRBuilder<> &B, Value *Offset) { +Value * +llvm::continuationStackOffsetToPtr(IRBuilder<> &B, Value *Offset, + Module &GpurtLibrary, + CompilerUtils::CrossModuleInliner &Inliner) { assert(Offset->getType()->isIntegerTy(32) && "Stack offset is expected to be an i32"); Module *M = B.GetInsertPoint()->getModule(); @@ -313,35 +313,19 @@ Value *llvm::continuationStackOffsetToPtr(IRBuilder<> &B, Value *Offset) { "Unexpected address space of the continuation stack"); auto *PtrTy = B.getInt8Ty()->getPointerTo(static_cast(*StackAddrspace)); - Value *BaseAddr = B.CreateCall(getContinuationStackGlobalMemBase(*M)); - BaseAddr = B.CreateIntToPtr(BaseAddr, PtrTy); - - return B.CreateGEP(B.getInt8Ty(), BaseAddr, Offset); -} - -Function *llvm::cloneFunctionHeader(Function &F, FunctionType *NewType, - ArrayRef ArgAttrs) { - LLVM_DEBUG(dbgs() << "Cloning function " << F.getName() << " with new type " - << *NewType << "\n"); - AttributeList FAttrs = F.getAttributes(); - AttributeList Attributes = AttributeList::get( - F.getContext(), FAttrs.getFnAttrs(), FAttrs.getRetAttrs(), ArgAttrs); - Function *NewFunc = Function::Create(NewType, F.getLinkage(), ""); - // Insert new function before F to facilitate writing tests - F.getParent()->getFunctionList().insert(F.getIterator(), NewFunc); - NewFunc->setCallingConv(F.getCallingConv()); - NewFunc->setSubprogram(F.getSubprogram()); - NewFunc->setDLLStorageClass(F.getDLLStorageClass()); - NewFunc->setAttributes(Attributes); - NewFunc->copyMetadata(&F, 0); - return NewFunc; + auto *BaseAddr = + Inliner.inlineCall(B, getContinuationStackGlobalMemBase(GpurtLibrary)) + .returnValue; + auto *BaseAddrPtr = B.CreateIntToPtr(BaseAddr, PtrTy); + + return B.CreateGEP(B.getInt8Ty(), BaseAddrPtr, Offset); } Function *llvm::cloneFunctionHeaderWithTypes(Function &F, DXILContFuncTy &NewType, ArrayRef ArgAttrs) { FunctionType *FuncTy = NewType.asFunctionType(F.getContext()); - Function *NewFunc = cloneFunctionHeader(F, FuncTy, ArgAttrs); + Function *NewFunc = CompilerUtils::cloneFunctionHeader(F, FuncTy, ArgAttrs); NewType.writeMetadata(NewFunc); return NewFunc; } @@ -552,10 +536,11 @@ Value *llvm::getDXILSystemData(IRBuilder<> &B, Value *SystemData, return B.CreateInBoundsGEP(OrigSystemDataTy, SystemData, Indices); } -CallInst *llvm::replaceIntrinsicCall(IRBuilder<> &B, Type *SystemDataTy, - Value *SystemData, DXILShaderKind Kind, - CallInst *Call) { - auto &M = *Call->getModule(); +CallInst * +llvm::replaceIntrinsicCall(IRBuilder<> &B, Type *SystemDataTy, + Value *SystemData, DXILShaderKind Kind, + CallInst *Call, Module *GpurtLibrary, + CompilerUtils::CrossModuleInliner &Inliner) { B.SetInsertPoint(Call); auto IntrImplEntry = findIntrImplEntryByIntrinsicCall(Call); @@ -563,11 +548,10 @@ CallInst *llvm::replaceIntrinsicCall(IRBuilder<> &B, Type *SystemDataTy, return nullptr; std::string Name = ("_cont_" + IntrImplEntry->Name).str(); - auto *IntrImpl = M.getFunction(Name); + auto *IntrImpl = GpurtLibrary->getFunction(Name); if (!IntrImpl) - cantFail(make_error(Twine("Intrinsic implementation '") + - Name + "' not found", - inconvertibleErrorCode())); + report_fatal_error(Twine("Intrinsic implementation '") + Name + + "' not found"); SmallVector Arguments; // Add the right system data type @@ -580,7 +564,8 @@ CallInst *llvm::replaceIntrinsicCall(IRBuilder<> &B, Type *SystemDataTy, Function *GetHitData; if (Kind == DXILShaderKind::AnyHit || Kind == DXILShaderKind::Intersection) { - auto *GetCandidateState = M.getFunction("_cont_GetCandidateState"); + auto *GetCandidateState = + GpurtLibrary->getFunction("_cont_GetCandidateState"); assert(GetCandidateState && "Could not find GetCandidateState function"); assert( GetCandidateState->getReturnType()->isStructTy() && @@ -590,7 +575,8 @@ CallInst *llvm::replaceIntrinsicCall(IRBuilder<> &B, Type *SystemDataTy, GetCandidateState->getFunctionType()->getParamType(0)->isPointerTy()); GetHitData = GetCandidateState; } else { - auto *GetCommittedState = M.getFunction("_cont_GetCommittedState"); + auto *GetCommittedState = + GpurtLibrary->getFunction("_cont_GetCommittedState"); assert(GetCommittedState && "Could not find GetCommittedState function"); assert( GetCommittedState->getReturnType()->isStructTy() && @@ -605,10 +591,13 @@ CallInst *llvm::replaceIntrinsicCall(IRBuilder<> &B, Type *SystemDataTy, B.SetInsertPoint(&*Call->getFunction()->begin()->begin()); auto *HitDataAlloca = B.CreateAlloca(GetHitData->getReturnType()); B.restoreIP(IP); - auto *HitData = B.CreateCall( - GetHitData, - {getDXILSystemData(B, SystemData, SystemDataTy, - getFuncArgPtrElementType(GetHitData, 0))}); + auto *HitData = + Inliner + .inlineCall( + B, GetHitData, + {getDXILSystemData(B, SystemData, SystemDataTy, + getFuncArgPtrElementType(GetHitData, 0))}) + .returnValue; B.CreateStore(HitData, HitDataAlloca); Arguments.push_back(HitDataAlloca); } @@ -634,10 +623,8 @@ CallInst *llvm::replaceIntrinsicCall(IRBuilder<> &B, Type *SystemDataTy, raw_string_ostream ToStream(To); ArgType->print(FromStream, true); NewType->print(ToStream, true); - cantFail(make_error(Twine("Can't convert ") + From + " to " + - To + " for intrinsic '" + - IntrImplEntry->Name + "'", - inconvertibleErrorCode())); + report_fatal_error(Twine("Can't convert ") + From + " to " + To + + " for intrinsic '" + IntrImplEntry->Name + "'"); } } @@ -646,6 +633,8 @@ CallInst *llvm::replaceIntrinsicCall(IRBuilder<> &B, Type *SystemDataTy, LLVM_DEBUG(dbgs() << "Replacing " << *Call << " by " << *NewCall << "\n"); if (!Call->getType()->isVoidTy()) Call->replaceAllUsesWith(NewCall); + Inliner.inlineCall(*NewCall); + B.SetInsertPoint(&*B.GetInsertPoint()); Call->eraseFromParent(); return NewCall; } @@ -756,16 +745,16 @@ bool llvm::earlyDriverTransform(Module &M) { replaceEnqueueIntrinsic(F, Replacement); } - if (Name == "_AmdContinuationStackIsGlobal") { + if (Name.starts_with("_AmdContinuationStackIsGlobal")) { Changed = true; handleContinuationStackIsGlobal(F, StackAddrspace); - } else if (Name == "_AmdContinuationsGetFlags") { + } else if (Name.starts_with("_AmdContinuationsGetFlags")) { Changed = true; if (!Flags) report_fatal_error("Tried to get continuation flags but it is not " "available on the module"); handleContinuationsGetFlags(F, *Flags); - } else if (Name == "_AmdGetRtip") { + } else if (Name.starts_with("_AmdGetRtip")) { Changed = true; if (!RtipLevel) report_fatal_error( @@ -842,7 +831,7 @@ bool llvm::LgcMaterializable(Instruction &OrigI) { // be rematerialized are replaced by their implementation, so that the // necessary values can be put into the coroutine frame. Therefore, we // can assume all left-over intrinsics can be rematerialized. - if (isRematerializableLgcRtOp(*CInst)) + if (DXILContHelper::isRematerializableLgcRtOp(*CInst)) return true; auto CalledName = CalledFunc->getName(); @@ -858,12 +847,8 @@ bool llvm::LgcMaterializable(Instruction &OrigI) { namespace llvm { void addLgcContinuationTransform(ModulePassManager &MPM) { - // Inline TraceRay and similar intrinsic implementations MPM.addPass(AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/false)); - // Lower GetShaderKind and GetCurrentFuncAddr - MPM.addPass(PreCoroutineLoweringPass()); - MPM.addPass(LowerAwaitPass()); MPM.addPass(CoroEarlyPass()); diff --git a/shared/continuations/lib/DXILContIntrinsicPrepare.cpp b/shared/continuations/lib/DXILContIntrinsicPrepare.cpp index ef8b6aae08..a58d91f53c 100644 --- a/shared/continuations/lib/DXILContIntrinsicPrepare.cpp +++ b/shared/continuations/lib/DXILContIntrinsicPrepare.cpp @@ -33,13 +33,14 @@ //===----------------------------------------------------------------------===// #include "continuations/Continuations.h" +#include "continuations/ContinuationsUtil.h" +#include "lgc/LgcRtDialect.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/InitializePasses.h" -#include #include #include @@ -57,6 +58,7 @@ static Function *transformFunction(Function &F) { auto Name = F.getName(); LLVM_DEBUG(dbgs() << "Transforming function " << Name << "\n"); std::string NewName = Name.str(); + // Unmangle declarations because they cannot be renamed in the dx api if (Name.contains('@')) { // Extract unmangled name @@ -64,10 +66,9 @@ static Function *transformFunction(Function &F) { auto End = Name.find('@', Start); if (Start == 0 || End == StringRef::npos || Start > Name.size() || End > Name.size()) { - cantFail(make_error( + report_fatal_error( Twine("Failed to unmangle function name: Failed to extract from '") + - Name + "' (start: " + Twine(Start) + ", end: " + Twine(End) + ")", - inconvertibleErrorCode())); + Name + "' (start: " + Twine(Start) + ", end: " + Twine(End) + ")"); } // Copy name, otherwise it will be deleted before it's set @@ -84,54 +85,67 @@ static Function *transformFunction(Function &F) { // Unpack the inner type of @class.matrix types bool UnpackMatrixTy = false; - static const std::array FuncsReturningMatrices = { - "ObjectToWorld4x3", "WorldToObject4x3"}; - if (NewRetTy->isStructTy() && NewRetTy->getStructNumElements() == 1) { - StringRef FuncName = F.getName(); - for (auto FuncCandidate : FuncsReturningMatrices) { - if (FuncName.contains(FuncCandidate)) { - NewRetTy = NewRetTy->getStructElementType(0); - UnpackMatrixTy = true; - break; - } + if (Name.contains("ObjectToWorld4x3") || + Name.contains("WorldToObject4x3")) { + NewRetTy = NewRetTy->getStructElementType(0); + UnpackMatrixTy = true; } } + // TODO Remove old name when possible + if (NewName == "_cont_Traversal" || Name == "amd.dx.TraversalImpl") + lgc::rt::setLgcRtShaderStage(&F, lgc::rt::RayTracingShaderStage::Traversal); + Argument *RetArg = nullptr; AttributeList FAttrs = F.getAttributes(); SmallVector ParamAttrs; + unsigned ArgNo = 0; for (auto &Arg : F.args()) { DXILContArgTy ArgTy = DXILContArgTy::get(&F, &Arg); + + bool DidHandleArg = false; + if (Arg.hasStructRetAttr()) { NewRetTy = Arg.getParamStructRetType(); RetArg = &Arg; - } else if (Arg.getType()->isPointerTy() && - (StringRef(NewName).contains("Await") || - StringRef(NewName).contains("Enqueue") || - StringRef(NewName).contains("Traversal") || - (NewName == "_cont_SetTriangleHitAttributes" && - &Arg != F.getArg(0)))) { - // Pass argument data as struct instead of as pointer - Type *ElemType = ArgTy.getPointerElementType(); - assert(ElemType && "unable to resolve pointer type for argument"); - AllArgTypes.emplace_back(ElemType); - ParamAttrs.push_back({}); - } else { + + DidHandleArg = true; + } else if (Arg.getType()->isPointerTy()) { + StringRef NameRef{NewName}; + if (NameRef.contains("Await") || NameRef.contains("Enqueue") || + NameRef.contains("Traversal") || + (NewName == "_cont_SetTriangleHitAttributes" && + &Arg != F.getArg(0))) { + // Pass argument data as struct instead of as pointer + Type *ElemType = ArgTy.getPointerElementType(); + assert(ElemType && "Unable to resolve pointer type for argument"); + AllArgTypes.emplace_back(ElemType); + ParamAttrs.push_back({}); + + DidHandleArg = true; + } + } + + // Simply add the argument and its type. + if (!DidHandleArg) { AllArgTypes.push_back(ArgTy); ParamAttrs.push_back(FAttrs.getParamAttrs(ArgNo)); } + ArgNo++; } // Create new empty function DXILContFuncTy NewFuncTy(NewRetTy, AllArgTypes); Function *NewFunc = cloneFunctionHeaderWithTypes(F, NewFuncTy, ParamAttrs); + // Remove old name for the case that the new name is the same F.setName(""); NewFunc->setName(NewName); NewFunc->addFnAttr(Attribute::AlwaysInline); + // Set external linkage, so the functions don't get removed, even if they are // never referenced at this point NewFunc->setLinkage(GlobalValue::LinkageTypes::ExternalLinkage); @@ -141,21 +155,23 @@ static Function *transformFunction(Function &F) { // Do not insert code on function declarations std::optional> B; - if (!NewFunc->empty()) + bool IsDeclaration = NewFunc->empty(); + + if (!IsDeclaration) { B.emplace(&*NewFunc->getEntryBlock().getFirstNonPHIOrDbgOrAlloca()); - if (B && UnpackMatrixTy) { - // Move values of @class.matrix.x.y into return value of unpacked type - // Replace the return instruction with a new one, returning the unpacked - // value - for (auto &BB : *NewFunc) { - auto *I = BB.getTerminator(); - if (I->getOpcode() == Instruction::Ret) { - B->SetInsertPoint(I); - Value *UnpackedVal = B->CreateExtractValue(I->getOperand(0), {0}); - B->CreateRet(UnpackedVal); - I->eraseFromParent(); - } + if (UnpackMatrixTy) { + // Move values of @class.matrix.x.y into return value of unpacked type + // Replace the return instruction with a new one, returning the unpacked + // value + llvm::forEachTerminator( + NewFunc, {Instruction::Ret}, [&](Instruction &Terminator) { + B->SetInsertPoint(&Terminator); + Value *RetExtractVal = + B->CreateExtractValue(Terminator.getOperand(0), {0}); + B->CreateRet(RetExtractVal); + Terminator.eraseFromParent(); + }); } } @@ -174,7 +190,8 @@ static Function *transformFunction(Function &F) { Argument *Arg = NewFunc->getArg(NewIdx); Arg->setName(OldArg->getName()); - if (B) { + + if (!IsDeclaration) { if (Arg->getType() != OldArg->getType()) { // Replace pointer argument with alloca auto *Ty = Arg->getType(); @@ -195,32 +212,25 @@ static Function *transformFunction(Function &F) { Arg->removeAttr(Attribute::AttrKind::InReg); } - if (RetArg && B) { + if (RetArg && !IsDeclaration) { // Replace sret argument with real return value B->SetInsertPoint(&*NewFunc->getEntryBlock().getFirstNonPHIOrDbgOrAlloca()); auto *RetAlloca = B->CreateAlloca(NewRetTy); RetArg->replaceAllUsesWith(RetAlloca); // Replace returns with return value - for (auto &BB : *NewFunc) { - auto *I = BB.getTerminator(); - if (I->getOpcode() == Instruction::Ret) { - B->SetInsertPoint(I); - auto *RetLoad = B->CreateLoad(NewRetTy, RetAlloca); - B->CreateRet(RetLoad); - I->eraseFromParent(); - } - } + llvm::forEachTerminator( + NewFunc, {Instruction::Ret}, [&](Instruction &Terminator) { + B->SetInsertPoint(&Terminator); + Value *RetLoad = B->CreateLoad(NewRetTy, RetAlloca); + B->CreateRet(RetLoad); + Terminator.eraseFromParent(); + }); } // Replace all calls SmallVector Uses; - for (auto &Use : F.uses()) { - if (auto *CInst = dyn_cast(Use.getUser())) { - if (CInst->isCallee(&Use)) - Uses.push_back(CInst); - } - } + llvm::forEachCall(F, [&](CallInst &CInst) { Uses.push_back(&CInst); }); for (auto *CInst : Uses) { if (!B) @@ -303,6 +313,7 @@ static bool isUtilFunction(StringRef Name) { if (Name.contains(UtilName)) return true; } + return false; } @@ -311,12 +322,22 @@ llvm::PreservedAnalyses DXILContIntrinsicPreparePass::run( LLVM_DEBUG(dbgs() << "Run the dxil-cont-intrinsic-prepare pass\n"); SmallVector Funcs(make_pointer_range(M.functions())); + for (auto *F : Funcs) { auto Name = F->getName(); bool IsContImpl = Name.contains("_cont_") || Name.contains("amd.dx."); - bool IsAmdIntr = Name.contains("_Amd"); - if ((IsContImpl && isGpuRtFuncName(Name)) || - ((IsContImpl || IsAmdIntr) && isUtilFunction(Name))) + bool ShouldTransform = false; + + if (IsContImpl) { + if (isGpuRtFuncName(Name)) + ShouldTransform = true; + else if (isUtilFunction(Name)) + ShouldTransform = true; + } else if (Name.contains("_Amd") && isUtilFunction(Name)) { + ShouldTransform = true; + } + + if (ShouldTransform) transformFunction(*F); } diff --git a/shared/continuations/lib/DXILContLgcRtOpConverter.cpp b/shared/continuations/lib/DXILContLgcRtOpConverter.cpp index f8e2df799e..993a38bde1 100644 --- a/shared/continuations/lib/DXILContLgcRtOpConverter.cpp +++ b/shared/continuations/lib/DXILContLgcRtOpConverter.cpp @@ -499,6 +499,11 @@ void DXILContLgcRtOpConverterPass::applyPayloadMetadataTypesOnShaders() { for (auto &[Func, Kind] : ShaderKinds) { auto Stage = DXILContHelper::dxilShaderKindToShaderStage(Kind); + + // Ignore non-raytracing shader stages + if (!Stage.has_value()) + continue; + lgc::rt::setLgcRtShaderStage(Func, Stage); switch (Kind) { diff --git a/shared/continuations/lib/DXILContPostProcess.cpp b/shared/continuations/lib/DXILContPostProcess.cpp index 81567b40ee..0925f614cb 100644 --- a/shared/continuations/lib/DXILContPostProcess.cpp +++ b/shared/continuations/lib/DXILContPostProcess.cpp @@ -36,10 +36,13 @@ // //===----------------------------------------------------------------------===// +#include "compilerutils/CompilerUtils.h" #include "continuations/Continuations.h" #include "continuations/ContinuationsDialect.h" #include "continuations/ContinuationsUtil.h" #include "llvm-dialects/Dialect/Visitor.h" +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/Analysis/AssumptionCache.h" @@ -48,6 +51,7 @@ #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" #include "llvm/IR/Type.h" #include "llvm/InitializePasses.h" #include @@ -75,17 +79,56 @@ static cl::opt ReportAllSizes( cl::desc("Report continuation state, payload and system data sizes."), cl::init(false)); -// Collects all calls to the given function, and appends them to CallInsts. -static void collectFunctionCalls(const Function &Func, - SmallVectorImpl &CallInsts) { - for (const auto &Use : Func.uses()) { - if (auto *CInst = dyn_cast(Use.getUser())) { - if (CInst->isCallee(&Use)) { - CallInsts.push_back(CInst); - } - } - } -} +namespace { +class DXILContPostProcessPassImpl final { +public: + DXILContPostProcessPassImpl(Module &M, Module &GpurtLibrary); + bool run(llvm::ModuleAnalysisManager &AnalysisManager); + + struct FunctionData { + DXILShaderKind Kind = DXILShaderKind::Invalid; + /// Calls to hlsl intrinsics + SmallVector IntrinsicCalls; + /// Calls to get the system data pointer + SmallVector GetSystemDataCalls; + + /// If this is the start function part of a split function + bool IsStart = true; + /// Pointer to the alloca'd system data object in this function + Value *SystemData = nullptr; + Type *SystemDataTy = nullptr; + }; + +private: + void lowerGetResumePointAddr(Function &F); + void handleInitialContinuationStackPtr(Function &F); + void handleLgcRtIntrinsic(Function &F); + void handleRegisterBufferSetPointerBarrier(Function &F, + GlobalVariable *Payload); + void handleRegisterBufferGetPointer(Function &F, GlobalVariable *Payload); + void handleValueI32Count(Function &F); + void handleValueGetI32(Function &F); + void handleValueSetI32(Function &F); + + void handleContPayloadRegisterI32Count(Function &F); + void handleContPayloadRegistersGetI32(Function &F); + void handleContPayloadRegistersSetI32(Function &F); + void handleContStackAlloc(FunctionAnalysisManager &FAM, Function &F); + + void collectProcessableFunctions(); + bool handleIntrinsicCalls(); + bool handleGetSystemDataCalls(); + bool unfoldGlobals(); + bool handleAmdInternals(llvm::ModuleAnalysisManager &AnalysisManager); + + Module *Mod; + Module *GpurtLibrary; + GlobalVariable *Registers; + MapVector ToProcess; + Function *SetupRayGen; + IRBuilder<> Builder; + CompilerUtils::CrossModuleInliner CrossInliner; +}; // Collects all calls to continuation.[wait]continue static void collectContinueCalls(const Module &M, @@ -95,7 +138,9 @@ static void collectContinueCalls(const Module &M, auto *Func = M.getFunction(Name); if (!Func) continue; - collectFunctionCalls(*Func, CallInsts); + + llvm::forEachCall(*Func, + [&](CallInst &CInst) { CallInsts.push_back(&CInst); }); } } @@ -136,7 +181,6 @@ static void reportContStateSizes(Module &M) { // For every function with incoming or outgoing (or both) payload registers, // report the incoming size and the max outgoing size in bytes. static void reportPayloadSizes(Module &M) { - // For every function with continue calls, determine the max number of // outgoing registers DenseMap MaxOutgoingRegisterCounts; @@ -185,7 +229,7 @@ static void reportPayloadSizes(Module &M) { static void reportSystemDataSizes( Module &M, - const MapVector + const MapVector &FunctionData) { for (const auto &[F, FuncData] : FunctionData) { if (FuncData.SystemDataTy == nullptr) @@ -200,8 +244,6 @@ static void reportSystemDataSizes( } } -DXILContPostProcessPass::DXILContPostProcessPass() {} - static Function *getContinuationGetAddrAndMD(Module &M) { auto *Name = "continuation.getAddrAndMD"; if (auto *F = M.getFunction(Name)) @@ -328,20 +370,16 @@ static bool addGetAddrAndMDIntrinsicCalls(Module &M) { } // Check that every function has at most one setLocalRootIndex call. - if (const auto *SetF = M.getFunction("amd.dx.setLocalRootIndex")) { + if (auto *SetF = M.getFunction("amd.dx.setLocalRootIndex")) { SmallDenseSet HasSetF; - for (const auto &Use : SetF->uses()) { - if (auto *CInst = dyn_cast(Use.getUser())) { - if (CInst->isCallee(&Use)) { - // Returns true if it is a new value - auto Inserted = HasSetF.insert(CInst->getFunction()); - if (!Inserted.second) - report_fatal_error( - "Found a function with more than one setLocalRootIndex"); - } - } - } + llvm::forEachCall(*SetF, [&](CallInst &CInst) { + // Returns true if it is a new value + auto Inserted = HasSetF.insert(CInst.getFunction()); + if (!Inserted.second) + report_fatal_error( + "Found a function with more than one setLocalRootIndex"); + }); } } @@ -432,13 +470,8 @@ static std::optional findContinueCall(CallInst *GetResPointAddr) { return Candidate; } -bool DXILContPostProcessPass::lowerGetResumePointAddr( - llvm::Module &M, llvm::IRBuilder<> &B, - const MapVector &ToProcess) { - auto *GetResumePointAddr = M.getFunction("_AmdGetResumePointAddr"); - - if (!GetResumePointAddr) - return false; +void DXILContPostProcessPassImpl::lowerGetResumePointAddr(Function &F) { + auto *GetResumePointAddr = &F; assert(GetResumePointAddr->getReturnType()->isIntegerTy(64) && GetResumePointAddr->arg_size() == 0); @@ -487,7 +520,7 @@ bool DXILContPostProcessPass::lowerGetResumePointAddr( unsigned ReturnAddrArgNum = HasWaitMask ? 3 : 2; // Move up computation of the resume address auto *ReturnAddr = ContinueCall->getArgOperand(ReturnAddrArgNum); - assert((ReturnAddr->getType() == B.getInt64Ty()) && + assert((ReturnAddr->getType() == Builder.getInt64Ty()) && "Unexpected return addr type!"); SmallVector MoveInstrs; @@ -516,8 +549,8 @@ bool DXILContPostProcessPass::lowerGetResumePointAddr( Args.push_back(ContinueCall->getArgOperand(I)); } - B.SetInsertPoint(ContinueCall); - auto *NewCall = B.CreateCall(ContinueCall->getCalledFunction(), Args); + Builder.SetInsertPoint(ContinueCall); + auto *NewCall = Builder.CreateCall(ContinueCall->getCalledFunction(), Args); // Copy metadata SmallVector> MDs; ContinueCall->getAllMetadata(MDs); @@ -534,28 +567,24 @@ bool DXILContPostProcessPass::lowerGetResumePointAddr( // Delete the declaration of the intrinsic after lowering, as future calls to // it are invalid. GetResumePointAddr->eraseFromParent(); - - return true; } -void DXILContPostProcessPass::handleInitialContinuationStackPtr(IRBuilder<> &B, - Function &F) { - auto *InitFun = Mod->getFunction("_cont_GetContinuationStackAddr"); +void DXILContPostProcessPassImpl::handleInitialContinuationStackPtr( + Function &F) { + auto *InitFun = GpurtLibrary->getFunction("_cont_GetContinuationStackAddr"); assert(InitFun && "GetContinuationStackAddr not found"); assert(InitFun->arg_size() == 0 && InitFun->getReturnType()->isIntegerTy(32)); - for (auto &Use : make_early_inc_range(F.uses())) { - if (auto *CInst = dyn_cast(Use.getUser())) { - if (CInst->isCallee(&Use)) { - B.SetInsertPoint(CInst); - auto *Init = B.CreateCall(InitFun); - CInst->replaceAllUsesWith(Init); - CInst->eraseFromParent(); - } - } - } + llvm::forEachCall(F, [&](CallInst &CInst) { + Builder.SetInsertPoint(&CInst); + auto *Init = Builder.CreateCall(InitFun); + CInst.replaceAllUsesWith(Init); + CrossInliner.inlineCall(*Init); + Builder.SetInsertPoint(&*Builder.GetInsertPoint()); + CInst.eraseFromParent(); + }); } -void DXILContPostProcessPass::handleLgcRtIntrinsic(Function &F) { +void DXILContPostProcessPassImpl::handleLgcRtIntrinsic(Function &F) { // Look for known HLSL intrinsics llvm::forEachCall(F, [&](CallInst &CInst) { auto Data = ToProcess.find(CInst.getFunction()); @@ -569,47 +598,38 @@ void DXILContPostProcessPass::handleLgcRtIntrinsic(Function &F) { }); } -void DXILContPostProcessPass::handleRegisterBufferSetPointerBarrier( +void DXILContPostProcessPassImpl::handleRegisterBufferSetPointerBarrier( Function &F, GlobalVariable *Payload) { // Remove setpointerbarrier instructions related to payload - for (auto &Use : make_early_inc_range(F.uses())) { - if (auto *CInst = dyn_cast(Use.getUser())) { - if (CInst->isCallee(&Use)) { - if (!isCastGlobal(Payload, CInst->getOperand(0))) - continue; - CInst->eraseFromParent(); - } - } - } + llvm::forEachCall(F, [&](CallInst &CInst) { + if (isCastGlobal(Payload, CInst.getOperand(0))) + CInst.eraseFromParent(); + }); } -void DXILContPostProcessPass::handleRegisterBufferGetPointer( - IRBuilder<> &B, Function &F, GlobalVariable *Payload) { +void DXILContPostProcessPassImpl::handleRegisterBufferGetPointer( + Function &F, GlobalVariable *Payload) { // Check calls that take the payload as argument - for (auto &Use : make_early_inc_range(F.uses())) { - if (auto *CInst = dyn_cast(Use.getUser())) { - if (CInst->isCallee(&Use)) { - if (!isCastGlobal(Payload, CInst->getOperand(0))) - continue; - - // Replace call with first part of payload - static_assert(FirstPayloadMemoryPointerRegister == 0, - "Need to adjust offset here"); - B.SetInsertPoint(CInst); - auto *StackOffsetTy = getContinuationStackOffsetType(F.getContext()); - auto *CastPayload = B.CreateBitOrPointerCast( - Payload, StackOffsetTy->getPointerTo(Payload->getAddressSpace())); - auto *Offset = B.CreateLoad(StackOffsetTy, CastPayload); - auto *Ptr = continuationStackOffsetToPtr(B, Offset); - Ptr = B.CreateBitCast(Ptr, CInst->getType()); - CInst->replaceAllUsesWith(Ptr); - CInst->eraseFromParent(); - } + llvm::forEachCall(F, [&](CallInst &CInst) { + if (isCastGlobal(Payload, CInst.getOperand(0))) { + // Replace call with first part of payload + static_assert(FirstPayloadMemoryPointerRegister == 0, + "Need to adjust offset here"); + Builder.SetInsertPoint(&CInst); + auto *StackOffsetTy = getContinuationStackOffsetType(F.getContext()); + auto *CastPayload = Builder.CreateBitOrPointerCast( + Payload, StackOffsetTy->getPointerTo(Payload->getAddressSpace())); + auto *Offset = Builder.CreateLoad(StackOffsetTy, CastPayload); + auto *Ptr = continuationStackOffsetToPtr(Builder, Offset, *GpurtLibrary, + CrossInliner); + Ptr = Builder.CreateBitCast(Ptr, CInst.getType()); + CInst.replaceAllUsesWith(Ptr); + CInst.eraseFromParent(); } - } + }); } -void DXILContPostProcessPass::handleValueI32Count(IRBuilder<> &B, Function &F) { +void DXILContPostProcessPassImpl::handleValueI32Count(Function &F) { assert(F.arg_size() == 1 // i32 count && F.getFunctionType()->getReturnType()->isIntegerTy(32) @@ -617,19 +637,15 @@ void DXILContPostProcessPass::handleValueI32Count(IRBuilder<> &B, Function &F) { && F.getFunctionType()->getParamType(0)->isPointerTy()); auto *Ty = getFuncArgPtrElementType(&F, 0); - auto *Size = - B.getInt32(Mod->getDataLayout().getTypeStoreSize(Ty).getFixedValue() / 4); - for (auto &Use : make_early_inc_range(F.uses())) { - if (auto *CInst = dyn_cast(Use.getUser())) { - if (CInst->isCallee(&Use)) { - CInst->replaceAllUsesWith(Size); - CInst->eraseFromParent(); - } - } - } + auto *Size = Builder.getInt32( + Mod->getDataLayout().getTypeStoreSize(Ty).getFixedValue() / 4); + llvm::forEachCall(F, [&](CallInst &CInst) { + CInst.replaceAllUsesWith(Size); + CInst.eraseFromParent(); + }); } -void DXILContPostProcessPass::handleValueGetI32(IRBuilder<> &B, Function &F) { +void DXILContPostProcessPassImpl::handleValueGetI32(Function &F) { assert(F.arg_size() == 2 // value && F.getFunctionType()->getReturnType()->isIntegerTy(32) @@ -638,23 +654,19 @@ void DXILContPostProcessPass::handleValueGetI32(IRBuilder<> &B, Function &F) { // index && F.getFunctionType()->getParamType(1)->isIntegerTy(32)); - auto *I32 = B.getInt32Ty(); - for (auto &Use : make_early_inc_range(F.uses())) { - if (auto *CInst = dyn_cast(Use.getUser())) { - if (CInst->isCallee(&Use)) { - B.SetInsertPoint(CInst); - Value *Addr = - B.CreateBitCast(CInst->getArgOperand(0), I32->getPointerTo()); - Addr = B.CreateGEP(I32, Addr, CInst->getArgOperand(1)); - auto *Load = B.CreateLoad(I32, Addr); - CInst->replaceAllUsesWith(Load); - CInst->eraseFromParent(); - } - } - } + auto *I32 = Builder.getInt32Ty(); + llvm::forEachCall(F, [&](CallInst &CInst) { + Builder.SetInsertPoint(&CInst); + Value *Addr = + Builder.CreateBitCast(CInst.getArgOperand(0), I32->getPointerTo()); + Addr = Builder.CreateGEP(I32, Addr, CInst.getArgOperand(1)); + auto *Load = Builder.CreateLoad(I32, Addr); + CInst.replaceAllUsesWith(Load); + CInst.eraseFromParent(); + }); } -void DXILContPostProcessPass::handleValueSetI32(IRBuilder<> &B, Function &F) { +void DXILContPostProcessPassImpl::handleValueSetI32(Function &F) { assert(F.arg_size() == 3 && F.getFunctionType()->getReturnType()->isVoidTy() // Pointer to a struct @@ -664,22 +676,19 @@ void DXILContPostProcessPass::handleValueSetI32(IRBuilder<> &B, Function &F) { // value && F.getFunctionType()->getParamType(2)->isIntegerTy(32)); - auto *I32 = B.getInt32Ty(); - for (auto &Use : make_early_inc_range(F.uses())) { - if (auto *CInst = dyn_cast(Use.getUser())) { - if (CInst->isCallee(&Use)) { - B.SetInsertPoint(CInst); - Value *Addr = - B.CreateBitCast(CInst->getArgOperand(0), I32->getPointerTo()); - Addr = B.CreateGEP(I32, Addr, CInst->getArgOperand(1)); - B.CreateStore(CInst->getArgOperand(2), Addr); - CInst->eraseFromParent(); - } - } - } + auto *I32 = Builder.getInt32Ty(); + llvm::forEachCall(F, [&](CallInst &CInst) { + Builder.SetInsertPoint(&CInst); + Value *Addr = + Builder.CreateBitCast(CInst.getArgOperand(0), I32->getPointerTo()); + Addr = Builder.CreateGEP(I32, Addr, CInst.getArgOperand(1)); + Builder.CreateStore(CInst.getArgOperand(2), Addr); + CInst.eraseFromParent(); + }); } -void DXILContPostProcessPass::handleContPayloadRegisterI32Count(Function &F) { +void DXILContPostProcessPassImpl::handleContPayloadRegisterI32Count( + Function &F) { assert(F.arg_size() == 0 // register count && F.getFunctionType()->getReturnType()->isIntegerTy(32)); @@ -687,39 +696,32 @@ void DXILContPostProcessPass::handleContPayloadRegisterI32Count(Function &F) { auto *RegCount = ConstantInt::get(IntegerType::get(F.getContext(), 32), Registers->getValueType()->getArrayNumElements()); - for (auto &Use : make_early_inc_range(F.uses())) { - if (auto *CInst = dyn_cast(Use.getUser())) { - if (CInst->isCallee(&Use)) { - CInst->replaceAllUsesWith(RegCount); - CInst->eraseFromParent(); - } - } - } + llvm::forEachCall(F, [&](CallInst &CInst) { + CInst.replaceAllUsesWith(RegCount); + CInst.eraseFromParent(); + }); } -void DXILContPostProcessPass::handleContPayloadRegistersGetI32(IRBuilder<> &B, - Function &F) { +void DXILContPostProcessPassImpl::handleContPayloadRegistersGetI32( + Function &F) { assert(F.getReturnType()->isIntegerTy(32) && F.arg_size() == 1 // index && F.getFunctionType()->getParamType(0)->isIntegerTy(32)); - for (auto &Use : make_early_inc_range(F.uses())) { - if (auto *CInst = dyn_cast(Use.getUser())) { - if (CInst->isCallee(&Use)) { - B.SetInsertPoint(CInst); - auto *Addr = B.CreateGEP(Registers->getValueType(), Registers, - {B.getInt32(0), CInst->getArgOperand(0)}); - auto *Load = B.CreateLoad(B.getInt32Ty(), Addr); - CInst->replaceAllUsesWith(Load); - CInst->eraseFromParent(); - } - } - } + llvm::forEachCall(F, [&](CallInst &CInst) { + Builder.SetInsertPoint(&CInst); + auto *Addr = + Builder.CreateGEP(Registers->getValueType(), Registers, + {Builder.getInt32(0), CInst.getArgOperand(0)}); + auto *Load = Builder.CreateLoad(Builder.getInt32Ty(), Addr); + CInst.replaceAllUsesWith(Load); + CInst.eraseFromParent(); + }); } -void DXILContPostProcessPass::handleContPayloadRegistersSetI32(IRBuilder<> &B, - Function &F) { +void DXILContPostProcessPassImpl::handleContPayloadRegistersSetI32( + Function &F) { assert(F.getReturnType()->isVoidTy() && F.arg_size() == 2 // index @@ -727,22 +729,18 @@ void DXILContPostProcessPass::handleContPayloadRegistersSetI32(IRBuilder<> &B, // value && F.getFunctionType()->getParamType(1)->isIntegerTy(32)); - for (auto &Use : make_early_inc_range(F.uses())) { - if (auto *CInst = dyn_cast(Use.getUser())) { - if (CInst->isCallee(&Use)) { - B.SetInsertPoint(CInst); - auto *Addr = B.CreateGEP(Registers->getValueType(), Registers, - {B.getInt32(0), CInst->getArgOperand(0)}); - B.CreateStore(CInst->getOperand(1), Addr); - CInst->eraseFromParent(); - } - } - } + llvm::forEachCall(F, [&](CallInst &CInst) { + Builder.SetInsertPoint(&CInst); + auto *Addr = + Builder.CreateGEP(Registers->getValueType(), Registers, + {Builder.getInt32(0), CInst.getArgOperand(0)}); + Builder.CreateStore(CInst.getOperand(1), Addr); + CInst.eraseFromParent(); + }); } -void DXILContPostProcessPass::handleContStackAlloc(FunctionAnalysisManager &FAM, - IRBuilder<> &B, - Function &F) { +void DXILContPostProcessPassImpl::handleContStackAlloc( + FunctionAnalysisManager &FAM, Function &F) { assert(F.getReturnType()->isIntegerTy(32) && F.arg_size() == 2 // csp @@ -750,63 +748,53 @@ void DXILContPostProcessPass::handleContStackAlloc(FunctionAnalysisManager &FAM, // size && F.getFunctionType()->getParamType(1)->isIntegerTy(32)); - for (auto &Use : make_early_inc_range(F.uses())) { - if (auto *CInst = dyn_cast(Use.getUser())) { - if (CInst->isCallee(&Use)) { - B.SetInsertPoint(CInst); - auto *Func = CInst->getFunction(); - Value *SizeArg = CInst->getArgOperand(1); - uint32_t Size; - - if (auto *I = dyn_cast(SizeArg)) { - // Do some basic constant-propagation - // This is needed because this pass just replaced the ValueI32Count - // and ContPayloadRegistersI32Count intrinsics and the allocated size - // usually depends on these values. - auto &DT = FAM.getResult(*Func); - auto &TLI = FAM.getResult(*Func); - auto &AC = FAM.getResult(*Func); - const SimplifyQuery SQ(Func->getParent()->getDataLayout(), &TLI, &DT, - &AC); - - if (auto *NewSize = simplifyInstruction(I, SQ)) - SizeArg = NewSize; - } + llvm::forEachCall(F, [&](CallInst &CInst) { + Builder.SetInsertPoint(&CInst); + auto *Func = CInst.getFunction(); + Value *SizeArg = CInst.getArgOperand(1); + uint32_t Size; + + if (auto *I = dyn_cast(SizeArg)) { + // Do some basic constant-propagation + // This is needed because this pass just replaced the ValueI32Count + // and ContPayloadRegistersI32Count intrinsics and the allocated size + // usually depends on these values. + auto &DT = FAM.getResult(*Func); + auto &TLI = FAM.getResult(*Func); + auto &AC = FAM.getResult(*Func); + const SimplifyQuery SQ(Func->getParent()->getDataLayout(), &TLI, &DT, + &AC); + + if (auto *NewSize = simplifyInstruction(I, SQ)) + SizeArg = NewSize; + } - if (auto *C = dyn_cast(SizeArg)) - Size = C->getZExtValue(); - else - report_fatal_error("ContStackAlloc must be called with a constant " - "that can be computed at compile time"); + if (auto *C = dyn_cast(SizeArg)) + Size = C->getZExtValue(); + else + report_fatal_error("ContStackAlloc must be called with a constant " + "that can be computed at compile time"); - auto *OrigVal = B.CreateLoad(B.getInt32Ty(), CInst->getArgOperand(0)); + auto *OrigVal = + Builder.CreateLoad(Builder.getInt32Ty(), CInst.getArgOperand(0)); - auto *NewVal = B.CreateAdd(OrigVal, B.getInt32(Size)); - B.CreateStore(NewVal, CInst->getArgOperand(0)); - CInst->replaceAllUsesWith(OrigVal); - CInst->eraseFromParent(); + auto *NewVal = Builder.CreateAdd(OrigVal, Builder.getInt32(Size)); + Builder.CreateStore(NewVal, CInst.getArgOperand(0)); + CInst.replaceAllUsesWith(OrigVal); + CInst.eraseFromParent(); - // Add allocation to the stack size of this function - DXILContHelper::addStackSize(Func, Size); - } - } - } + // Add allocation to the stack size of this function + DXILContHelper::addStackSize(Func, Size); + }); } -llvm::PreservedAnalyses -DXILContPostProcessPass::run(llvm::Module &M, - llvm::ModuleAnalysisManager &AnalysisManager) { - LLVM_DEBUG(dbgs() << "Run the dxil-cont-post-process pass\n"); - AnalysisManager.getResult(M); - - Mod = &M; - bool Changed = false; - ToProcess.clear(); - auto *SetupRayGen = M.getFunction("_cont_SetupRayGen"); +void DXILContPostProcessPassImpl::collectProcessableFunctions() { + for (Function &F : *Mod) { + if (F.isDeclaration()) + continue; - for (Function &F : M) { auto Stage = lgc::rt::getLgcRtShaderStage(&F); - if (!Stage || F.isDeclaration()) + if (!Stage) continue; // Handle entry functions first @@ -827,15 +815,12 @@ DXILContPostProcessPass::run(llvm::Module &M, case DXILShaderKind::ClosestHit: case DXILShaderKind::Miss: case DXILShaderKind::Callable: { - Changed = true; FunctionData Data; Data.Kind = Kind; if (Data.Kind == DXILShaderKind::RayGeneration) { assert(SetupRayGen && "Could not find SetupRayGen function"); Data.SystemDataTy = SetupRayGen->getReturnType(); } else { - assert(F.getFunctionType()->getNumParams() >= 3 && - "Cannot find system data type"); Data.SystemDataTy = F.getFunctionType()->getParamType(2); } ToProcess[&F] = Data; @@ -847,7 +832,7 @@ DXILContPostProcessPass::run(llvm::Module &M, } // Also find continuation parts of the functions - for (auto &F : M) { + for (auto &F : *Mod) { if (F.isDeclaration()) continue; if (auto *MD = dyn_cast_or_null( @@ -862,14 +847,18 @@ DXILContPostProcessPass::run(llvm::Module &M, } } } +} - IRBuilder<> B(M.getContext()); - auto *Payload = M.getGlobalVariable(DXILContHelper::GlobalPayloadName); - for (auto &F : M.functions()) { +bool DXILContPostProcessPassImpl::handleIntrinsicCalls() { + bool Changed = false; + auto *Payload = Mod->getGlobalVariable(DXILContHelper::GlobalPayloadName); + + // TODO: Dialectify. + for (auto &F : Mod->functions()) { auto Name = F.getName(); if (Name == "continuation.initialContinuationStackPtr") { Changed = true; - handleInitialContinuationStackPtr(B, F); + handleInitialContinuationStackPtr(F); } else if (Name.startswith("lgc.rt")) { Changed = true; handleLgcRtIntrinsic(F); @@ -878,10 +867,14 @@ DXILContPostProcessPass::run(llvm::Module &M, handleRegisterBufferSetPointerBarrier(F, Payload); } else if (Name.startswith("registerbuffer.getpointer")) { Changed = true; - handleRegisterBufferGetPointer(B, F, Payload); + handleRegisterBufferGetPointer(F, Payload); } } + return Changed; +} + +bool DXILContPostProcessPassImpl::handleGetSystemDataCalls() { const static auto Visitor = llvm_dialects::VisitorBuilder>() .setStrategy(llvm_dialects::VisitorStrategy::ByFunctionDeclaration) @@ -894,139 +887,182 @@ DXILContPostProcessPass::run(llvm::Module &M, }) .build(); - Visitor.visit(ToProcess, M); + Visitor.visit(ToProcess, *Mod); for (auto &FuncData : ToProcess) { auto &Data = FuncData.second; // Transform SystemData to alloca and load on every use - B.SetInsertPoint(FuncData.first->getEntryBlock().getFirstNonPHI()); - Data.SystemData = B.CreateAlloca(Data.SystemDataTy); + Builder.SetInsertPoint(FuncData.first->getEntryBlock().getFirstNonPHI()); + Data.SystemData = Builder.CreateAlloca(Data.SystemDataTy); // Replace intrinsic calls for (auto *Call : Data.IntrinsicCalls) - replaceIntrinsicCall(B, Data.SystemDataTy, Data.SystemData, Data.Kind, - Call); + replaceIntrinsicCall(Builder, Data.SystemDataTy, Data.SystemData, + Data.Kind, Call, GpurtLibrary, CrossInliner); // Replace calls to getSystemData for (auto *Call : Data.GetSystemDataCalls) { - B.SetInsertPoint(Call); + Builder.SetInsertPoint(Call); auto *SystemDataTy = Call->getFunctionType()->getReturnType(); - auto *SystemDataPtr = getDXILSystemData(B, Data.SystemData, + auto *SystemDataPtr = getDXILSystemData(Builder, Data.SystemData, Data.SystemDataTy, SystemDataTy); - auto *SystemData = B.CreateLoad(SystemDataTy, SystemDataPtr); + auto *SystemData = Builder.CreateLoad(SystemDataTy, SystemDataPtr); Call->replaceAllUsesWith(SystemData); Call->eraseFromParent(); } - B.SetInsertPoint( + Builder.SetInsertPoint( &*FuncData.first->getEntryBlock().getFirstNonPHIOrDbgOrAlloca()); if (FuncData.first->hasMetadata(DXILContHelper::MDEntryName)) { // Initialize system data for the start part of the entry shader - auto *TmpSystemData = B.CreateCall(SetupRayGen); - B.CreateStore(TmpSystemData, Data.SystemData); + auto *TmpSystemData = Builder.CreateCall(SetupRayGen); + Builder.CreateStore(TmpSystemData, Data.SystemData); + CrossInliner.inlineCall(*TmpSystemData); + Builder.SetInsertPoint(&*Builder.GetInsertPoint()); } else { // Initialize the new system data alloca with the passed argument. - B.CreateStore(FuncData.first->getArg(Data.IsStart ? 2 : 1), - Data.SystemData); + Builder.CreateStore(FuncData.first->getArg(Data.IsStart ? 2 : 1), + Data.SystemData); } Data.SystemData->setName("system.data"); } - Changed |= lowerGetResumePointAddr(M, B, ToProcess); + return !ToProcess.empty(); +} +bool DXILContPostProcessPassImpl::unfoldGlobals() { // Replace register globals with indices into a bigger global - const auto &DL = M.getDataLayout(); + const auto &DL = Mod->getDataLayout(); GlobalVariable *PayloadGlobal = - M.getGlobalVariable(DXILContHelper::GlobalPayloadName); - GlobalVariable *ContStateGlobal = - M.getGlobalVariable(DXILContHelper::GlobalContStateName); - - if (PayloadGlobal || ContStateGlobal) { - Changed = true; + Mod->getGlobalVariable(DXILContHelper::GlobalPayloadName); + if (PayloadGlobal) { // We use the maximum size for the continuation state and the actual size // for the payload, so that the offset of the payload stays the same, but // the global is only as big as necessary. uint32_t RequiredSize = - (PayloadGlobal->getValueType()->getArrayNumElements() + - ContinuationStateRegisterCount) * - RegisterBytes; + PayloadGlobal->getValueType()->getArrayNumElements() * RegisterBytes; // Put continuation state first, it's callee save so we need to have it // full in all cases. Payload can be truncated, so the backend is free to - // use registers that are unused in a function - - auto *I32 = Type::getInt32Ty(M.getContext()); + // use registers that are unused in a function. + auto *I32 = Type::getInt32Ty(Mod->getContext()); auto *RegistersTy = ArrayType::get(I32, RequiredSize / RegisterBytes); - Registers = cast(M.getOrInsertGlobal( + Registers = cast(Mod->getOrInsertGlobal( DXILContHelper::GlobalRegistersName, RegistersTy, [&] { return new GlobalVariable( - M, RegistersTy, false, GlobalVariable::ExternalLinkage, nullptr, - DXILContHelper::GlobalRegistersName, nullptr, + *Mod, RegistersTy, false, GlobalVariable::ExternalLinkage, + nullptr, DXILContHelper::GlobalRegistersName, nullptr, GlobalVariable::NotThreadLocal, GlobalRegisterAddrspace); })); - if (ContStateGlobal) - replaceGlobal(DL, Registers, ContStateGlobal, 0); - if (PayloadGlobal) - replaceGlobal(DL, Registers, PayloadGlobal, - ContinuationStateRegisterCount * RegisterBytes); + replaceGlobal(DL, Registers, PayloadGlobal, 0); + + return true; } - Function *ContStackAlloc = nullptr; + return false; +} - for (auto &F : M.functions()) { +bool DXILContPostProcessPassImpl::handleAmdInternals( + llvm::ModuleAnalysisManager &AnalysisManager) { + bool Changed = false; + SmallVector ContStackAllocs; + + for (auto &F : Mod->functions()) { auto Name = F.getName(); if (Name.startswith("_AmdValueI32Count")) { Changed = true; - handleValueI32Count(B, F); + handleValueI32Count(F); } else if (Name.startswith("_AmdValueGetI32")) { Changed = true; - handleValueGetI32(B, F); + handleValueGetI32(F); } else if (Name.startswith("_AmdValueSetI32")) { Changed = true; - handleValueSetI32(B, F); - } else if (Name == "_AmdContPayloadRegistersI32Count") { + handleValueSetI32(F); + } else if (Name.starts_with("_AmdContPayloadRegistersI32Count")) { Changed = true; handleContPayloadRegisterI32Count(F); - } else if (Name == "_AmdContPayloadRegistersGetI32") { + } else if (Name.starts_with("_AmdContPayloadRegistersGetI32")) { Changed = true; - handleContPayloadRegistersGetI32(B, F); - } else if (Name == "_AmdContPayloadRegistersSetI32") { + handleContPayloadRegistersGetI32(F); + } else if (Name.starts_with("_AmdContPayloadRegistersSetI32")) { Changed = true; - handleContPayloadRegistersSetI32(B, F); - } else if (Name == "_AmdContStackAlloc") { + handleContPayloadRegistersSetI32(F); + } else if (Name.starts_with("_AmdContStackAlloc")) { Changed = true; - ContStackAlloc = &F; + ContStackAllocs.push_back(&F); } } - if (ContStackAlloc) { - auto &FAM = AnalysisManager.getResult(M) - .getManager(); - handleContStackAlloc(FAM, B, *ContStackAlloc); + if (!ContStackAllocs.empty()) { + auto &FAM = + AnalysisManager.getResult(*Mod) + .getManager(); + for (auto *F : ContStackAllocs) + handleContStackAlloc(FAM, *F); + } + + return Changed; +} + +DXILContPostProcessPassImpl::DXILContPostProcessPassImpl(Module &M, + Module &GpurtLibrary) + : Mod{&M}, GpurtLibrary{&GpurtLibrary}, + SetupRayGen{GpurtLibrary.getFunction("_cont_SetupRayGen")}, + Builder{Mod->getContext()} {} + +bool DXILContPostProcessPassImpl::run( + llvm::ModuleAnalysisManager &AnalysisManager) { + bool Changed = false; + + collectProcessableFunctions(); + + Changed |= handleIntrinsicCalls(); + Changed |= handleGetSystemDataCalls(); + for (auto &F : make_early_inc_range(*Mod)) { + if (F.getName().starts_with("_AmdGetResumePointAddr")) { + Changed = true; + lowerGetResumePointAddr(F); + } } + Changed |= unfoldGlobals(); + Changed |= handleAmdInternals(AnalysisManager); - Changed |= fixupDxilMetadata(M); + Changed |= fixupDxilMetadata(*Mod); // Change function pointer accesses to include metadata - Changed |= addGetAddrAndMDIntrinsicCalls(M); + Changed |= addGetAddrAndMDIntrinsicCalls(*Mod); #ifndef NDEBUG - checkContinuationsModule(M); + checkContinuationsModule(*Mod); #endif if (ReportContStateSizes || ReportAllSizes) - reportContStateSizes(M); + reportContStateSizes(*Mod); + if (ReportPayloadRegisterSizes || ReportAllSizes) - reportPayloadSizes(M); + reportPayloadSizes(*Mod); + if (ReportSystemDataSizes || ReportAllSizes) - reportSystemDataSizes(M, ToProcess); + reportSystemDataSizes(*Mod, ToProcess); + + Changed |= llvm::removeUnusedFunctionDecls(Mod, false); + + return Changed; +} +} // anonymous namespace + +llvm::PreservedAnalyses +DXILContPostProcessPass::run(llvm::Module &Module, + llvm::ModuleAnalysisManager &AnalysisManager) { + LLVM_DEBUG(dbgs() << "Run the pass dxil-cont-post-process\n"); + AnalysisManager.getResult(Module); - Changed |= llvm::removeUnusedFunctionDecls(&M, false); + DXILContPostProcessPassImpl Impl{Module, + GpurtLibrary ? *GpurtLibrary : Module}; + bool Changed = Impl.run(AnalysisManager); - if (Changed) - return PreservedAnalyses::none(); - return PreservedAnalyses::all(); + return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); } diff --git a/shared/continuations/lib/DXILSupport.cpp b/shared/continuations/lib/DXILSupport.cpp index c1f59906a6..4678f6694e 100644 --- a/shared/continuations/lib/DXILSupport.cpp +++ b/shared/continuations/lib/DXILSupport.cpp @@ -179,7 +179,7 @@ bool llvm::DXILMaterializable(Instruction &OrigI) { // be rematerialized are replaced by their implementation, so that the // necessary values can be put into the coroutine frame. Therefore, we // can assume all left-over intrinsics can be rematerialized. - if (isRematerializableLgcRtOp(*CInst)) + if (DXILContHelper::isRematerializableLgcRtOp(*CInst)) return true; auto CalledName = CalledFunc->getName(); diff --git a/shared/continuations/lib/LegacyCleanupContinuations.cpp b/shared/continuations/lib/LegacyCleanupContinuations.cpp index 775e0c7b79..2713fd1889 100644 --- a/shared/continuations/lib/LegacyCleanupContinuations.cpp +++ b/shared/continuations/lib/LegacyCleanupContinuations.cpp @@ -36,8 +36,10 @@ // //===----------------------------------------------------------------------===// +#include "compilerutils/CompilerUtils.h" #include "continuations/Continuations.h" #include "continuations/ContinuationsDialect.h" +#include "lgc/LgcRtDialect.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/IR/IRBuilder.h" @@ -52,14 +54,70 @@ using namespace llvm; #define DEBUG_TYPE "legacy-cleanup-continuations" -LegacyCleanupContinuationsPass::LegacyCleanupContinuationsPass() {} +namespace { + +class LegacyCleanupContinuationsPassImpl { +public: + LegacyCleanupContinuationsPassImpl( + llvm::Module &Mod, llvm::Module *GpurtLibrary, + llvm::ModuleAnalysisManager &AnalysisManager); + + llvm::PreservedAnalyses run(); + +private: + struct ContinuationData { + /// All functions belonging to this continuation, the entry function is the + /// first one + SmallVector Functions; + /// Size of the continuation state in byte + uint32_t ContStateBytes = 0; + CallInst *MallocCall = nullptr; + MDNode *MD = nullptr; + AllocaInst *NewContState = nullptr; + SmallVector NewReturnContinues; + /// Cleaned entry function, used to replace metadata + Function *NewStart = nullptr; + + // Returns the number of bytes used on the CPS stack for the continuation + // state. + uint32_t getContStateStackBytes() const { + return alignTo(ContStateBytes, RegisterBytes); + } + }; + + void analyzeContinuation(Function &F, MDNode *MD); + // Run analysis parts that need to wait until all resume functions have been + // collected + void finalizeContinuationData(Function &StartFunc, ContinuationData &Data); + void processContinuation(Function *StartFunc, ContinuationData &FuncData); + void handleFunctionEntry(ContinuationData &Data, Function *F, bool IsEntry); + void handleContinue(ContinuationData &Data, Instruction *Ret); + void handleSingleContinue(ContinuationData &Data, CallInst *Call, + Value *ResumeFun); + void handleReturn(ContinuationData &Data, CallInst *ContRet); + + Module &M; + LLVMContext &Context; + IRBuilder<> B; + Type *I32 = nullptr; + Type *I64 = nullptr; + Function *ContMalloc = nullptr; + Function *ContFree = nullptr; + Function *Continue = nullptr; + Function *WaitContinue = nullptr; + Function *Complete = nullptr; + MapVector ToProcess; + uint32_t MaxContStateBytes = 0; + llvm::Module *GpurtLibrary = nullptr; + CompilerUtils::CrossModuleInliner CrossInliner; +}; /// Find the original call that created the continuation token and the matching /// resume function for a return value. /// /// Returns a map (origin BB, (call that created the continuation token, resume /// function)). -static DenseMap> +DenseMap> findTokenOrigin(BasicBlock *BB, Value *V, SmallVectorImpl &ToRemove) { DenseMap> Result; @@ -127,7 +185,7 @@ findTokenOrigin(BasicBlock *BB, Value *V, } /// Create a memcopy of an array, which the translator understands -static void createCopy(IRBuilder<> &B, Value *Dst, Value *Src, Type *Ty) { +void createCopy(IRBuilder<> &B, Value *Dst, Value *Src, Type *Ty) { assert(Ty->isArrayTy() && "Can only copy arrays"); for (unsigned I = 0; I < Ty->getArrayNumElements(); I++) { auto *SrcGep = B.CreateConstInBoundsGEP2_32(Ty, Src, 0, I); @@ -137,8 +195,8 @@ static void createCopy(IRBuilder<> &B, Value *Dst, Value *Src, Type *Ty) { } } -void LegacyCleanupContinuationsPass::analyzeContinuation(Function &F, - MDNode *MD) { +void LegacyCleanupContinuationsPassImpl::analyzeContinuation(Function &F, + MDNode *MD) { // Only analyze main continuation auto *MDTup = cast(MD); auto *EntryF = mdconst::extract(MDTup->getOperand(0)); @@ -154,14 +212,11 @@ void LegacyCleanupContinuationsPass::analyzeContinuation(Function &F, // Search the malloc call to find the size of the continuation state if (ContMalloc) { - for (auto *User : ContMalloc->users()) { - if (auto *Call = dyn_cast(User)) { - if (Call->getFunction() == &F) { - Data.MallocCall = Call; - break; - } + forEachCall(*ContMalloc, [&](CallInst &Call) { + if (Call.getFunction() == &F) { + Data.MallocCall = &Call; } - } + }); } // Without malloc call, we check later if the continuation state is used @@ -169,281 +224,312 @@ void LegacyCleanupContinuationsPass::analyzeContinuation(Function &F, Data.ContStateBytes = cast(Data.MallocCall->getArgOperand(0))->getSExtValue(); } - if (Data.ContStateBytes > MaxContStateBytes) - MaxContStateBytes = Data.ContStateBytes; } -void LegacyCleanupContinuationsPass::processContinuations() { - auto &Context = M->getContext(); - auto *Void = Type::getVoidTy(Context); - - for (auto &FuncData : ToProcess) { - LLVM_DEBUG(dbgs() << "Processing function: " << FuncData.first->getName() - << "\n"); - bool IsEntry = FuncData.first->hasMetadata(DXILContHelper::MDEntryName); - - for (auto *F : FuncData.second.Functions) { - if (F != FuncData.first) { - // Entry marker should only be on the start and not on resume functions - F->eraseMetadata(Context.getMDKindID(DXILContHelper::MDEntryName)); - // Same for stacksize - F->eraseMetadata(Context.getMDKindID(DXILContHelper::MDStackSizeName)); - // Set same linkage as for start function - F->setLinkage(FuncData.first->getLinkage()); - } +void LegacyCleanupContinuationsPassImpl::finalizeContinuationData( + Function &StartFunc, ContinuationData &FuncData) { + if (FuncData.MallocCall) + return; + for (auto *F : FuncData.Functions) { + bool IsStart = (F == &StartFunc); // If this is the continuation start + Value *ContFrame; + if (IsStart) + ContFrame = F->getArg(F->arg_size() - 1); + else + ContFrame = F->getArg(0); + // If there are uses, we need to assume a size of + // MinimumContinuationStateBytes, because for all sizes up to this size + // coroutine passes will not emit a malloc that we can use to determine + // the exact size. If however the frame pointer is not used in any of + // the continuation functions, it's safe to assume an empty continuation + // state. + if (!ContFrame->user_empty()) { + assert(FuncData.ContStateBytes == 0); + FuncData.ContStateBytes = MinimumContinuationStateBytes; + break; + } + } +} - // Ignore the stub created for the coroutine passes - if (F->empty()) - continue; +// For a resume function, find the continue call to it (by looking at its uses) +// and obtain the incoming payload register count into the resume function +// as the outgoing register count of the continue call, indicated by metadata. +uint32_t getIncomingRegisterCount(Function *ResumeFunc) { + // For non-start functions, set (incoming) continuation registercount + // metadata by looking at the continue calls that reference this + // function. These continue calls both specify the number of their + // outgoing registers, and the number of incoming payload registers + // coming back into the resume function (i.e. us). + SmallVector Worklist(ResumeFunc->users()); + std::optional RegCount; + while (!Worklist.empty()) { + auto *U = Worklist.pop_back_val(); + if (auto *Const = dyn_cast(U)) { + Worklist.append(Const->user_begin(), Const->user_end()); + continue; + } + assert(isa(U) && + "User of a resume function should be a call to continue"); + auto *Inst = cast(U); + if (auto Count = DXILContHelper::tryGetReturnedRegisterCount(Inst)) { + assert((!RegCount || *RegCount == *Count) && + "Got different returned registercounts in continues to " + "the same resume function"); + RegCount = *Count; +#ifdef NDEBUG + break; +#endif + } else { + LLVM_DEBUG(Inst->dump()); + report_fatal_error("Found a continue call without " + "continuation returned registercount metadata"); + } + } + return RegCount.value(); +} - LLVM_DEBUG(dbgs() << "Processing function part: " << F->getName() - << "\n"); +Value *getContFrame(CallInst *MallocCall, Function *F, bool IsStart, + SmallVectorImpl &InstsToRemove) { + Value *ContFrame = nullptr; + if (MallocCall) { + if (IsStart) { + ContFrame = MallocCall; + InstsToRemove.push_back(MallocCall); + + auto *BufferArg = F->getArg(F->arg_size() - 1); + auto *User = BufferArg->getUniqueUndroppableUser(); + auto *Cast = dyn_cast(User); + if (Cast) + User = Cast->getUniqueUndroppableUser(); + auto *Store = cast(User); + InstsToRemove.push_back(Store); // Store needs to be eliminated first + if (Cast) + InstsToRemove.push_back(Cast); + } else { + // Look for the load of the allocated pointer + auto *User = F->getArg(0)->getUniqueUndroppableUser(); + auto *Cast = dyn_cast(User); + if (Cast) + User = Cast->getUniqueUndroppableUser(); + auto *Load = cast(User); + InstsToRemove.push_back(Load); // Load needs to be eliminated first + if (Cast) + InstsToRemove.push_back(Cast); + ContFrame = Load; + } + } else { + if (IsStart) + ContFrame = F->getArg(F->arg_size() - 1); + else + ContFrame = F->getArg(0); + } + return ContFrame; +} - bool IsStart = F == FuncData.first; // If this is the continuation start - SmallVector AllArgTypes; - SmallVector AllArgValues; - SmallVector InstsToRemove; - AttributeList FAttrs = F->getAttributes(); - SmallVector ParamAttrs; +void LegacyCleanupContinuationsPassImpl::processContinuation( + Function *StartFunc, ContinuationData &FuncData) { + auto *Void = Type::getVoidTy(Context); + LLVM_DEBUG(dbgs() << "Processing function: " << StartFunc->getName() << "\n"); + bool IsEntry = StartFunc->hasMetadata(DXILContHelper::MDEntryName); + // The start function must come first to setup FuncData.NewStart and + // ContMDTuple which is used by processing the resume functions. + assert(StartFunc == FuncData.Functions[0]); + MDTuple *ContMDTuple = nullptr; + + SmallVector ToRemove; + struct NewFunctionInfo { + Function *Func; + bool IsStart; + }; + SmallVector NewFuncs; + + for (auto *F : FuncData.Functions) { + if (F != StartFunc) { + // Entry marker should only be on the start and not on resume functions + F->eraseMetadata(Context.getMDKindID(DXILContHelper::MDEntryName)); + // Same for stacksize + F->eraseMetadata(Context.getMDKindID(DXILContHelper::MDStackSizeName)); + // Set same linkage as for start function + F->setLinkage(StartFunc->getLinkage()); + } - // Use all arguments except the last (pre-allocated buffer for the - // coroutine passes) for the continuation start - if (IsStart) { - unsigned ArgNo = 0; - assert(F->arg_size() >= 1 && - "Entry function has at least one argument"); - for (auto Arg = F->arg_begin(), ArgEnd = F->arg_end() - 1; - Arg != ArgEnd; Arg++) { - AllArgTypes.push_back(Arg->getType()); - AllArgValues.push_back(Arg); - ParamAttrs.push_back(FAttrs.getParamAttrs(ArgNo)); - ArgNo++; - } - } else { - IRBuilder<> B(&*F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca()); - AllArgTypes.push_back( - getContinuationStackOffsetType(Context)); // continuation stack ptr - AllArgValues.push_back(nullptr); - - // Find arguments from continuation.returnvalue calls - for (auto &I : F->getEntryBlock()) { - if (auto *Intr = dyn_cast(&I)) { - AllArgTypes.push_back(Intr->getType()); - AllArgValues.push_back(Intr); - InstsToRemove.push_back(Intr); - } - } + // Ignore the stub created for the coroutine passes + if (F->empty()) + return; + + LLVM_DEBUG(dbgs() << "Processing function part: " << F->getName() << "\n"); + + bool IsStart = F == StartFunc; // If this is the continuation start + SmallVector AllArgTypes; + SmallVector AllArgValues; + SmallVector InstsToRemove; + AttributeList FAttrs = F->getAttributes(); + SmallVector ParamAttrs; + + // Use all arguments except the last (pre-allocated buffer for the + // coroutine passes) for the continuation start + if (IsStart) { + unsigned ArgNo = 0; + assert(F->arg_size() >= 1 && "Entry function has at least one argument"); + for (auto Arg = F->arg_begin(), ArgEnd = F->arg_end() - 1; Arg != ArgEnd; + Arg++) { + AllArgTypes.push_back(Arg->getType()); + AllArgValues.push_back(Arg); + ParamAttrs.push_back(FAttrs.getParamAttrs(ArgNo)); + ArgNo++; } - - // Find the free call if there is one - if (ContFree) { - for (auto *User : ContFree->users()) { - if (auto *Call = dyn_cast(User)) { - if (Call->getFunction() == F) { - InstsToRemove.push_back(Call); - break; - } - } + } else { + B.SetInsertPoint(&*F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca()); + AllArgTypes.push_back( + getContinuationStackOffsetType(Context)); // continuation stack ptr + AllArgValues.push_back(nullptr); + + // Find arguments from continuation.returnvalue calls + for (auto &I : F->getEntryBlock()) { + if (auto *Intr = dyn_cast(&I)) { + AllArgTypes.push_back(Intr->getType()); + AllArgValues.push_back(Intr); + InstsToRemove.push_back(Intr); } } + } - // Find the continuation state pointer, either returned by the malloc or - // given as an argument - Value *ContFrame = nullptr; - if (FuncData.second.MallocCall) { - if (IsStart) { - ContFrame = FuncData.second.MallocCall; - InstsToRemove.push_back(FuncData.second.MallocCall); - - auto *BufferArg = F->getArg(F->arg_size() - 1); - auto *User = BufferArg->getUniqueUndroppableUser(); - auto *Cast = dyn_cast(User); - if (Cast) - User = Cast->getUniqueUndroppableUser(); - auto *Store = cast(User); - InstsToRemove.push_back(Store); // Store needs to be eliminated first - if (Cast) - InstsToRemove.push_back(Cast); - } else { - // Look for the load of the allocated pointer - auto *User = F->getArg(0)->getUniqueUndroppableUser(); - auto *Cast = dyn_cast(User); - if (Cast) - User = Cast->getUniqueUndroppableUser(); - auto *Load = cast(User); - InstsToRemove.push_back(Load); // Load needs to be eliminated first - if (Cast) - InstsToRemove.push_back(Cast); - ContFrame = Load; - } - } else { - if (IsStart) - ContFrame = F->getArg(F->arg_size() - 1); - else - ContFrame = F->getArg(0); - } + // Find the free call if there is one + if (ContFree) { + forEachCall(*ContFree, + [&](CallInst &CI) { InstsToRemove.push_back(&CI); }); + } - // Create new empty function - F->eraseMetadata(FuncData.second.MD->getMetadataID()); - auto *NewFuncTy = FunctionType::get(Void, AllArgTypes, false); - Function *NewFunc = cloneFunctionHeader(*F, NewFuncTy, ParamAttrs); - NewFunc->takeName(F); - FuncData.second.NewFunctions.push_back(NewFunc); - - // Transfer code from old function to new function - llvm::moveFunctionBody(*F, *NewFunc); - - // Set arg names for new function - for (unsigned Idx = 0; Idx != NewFunc->getFunctionType()->params().size(); - ++Idx) { - Argument *Arg = NewFunc->getArg(Idx); - Value *OldVal = AllArgValues[Idx]; - if (OldVal) { - Arg->setName(OldVal->getName()); - OldVal->replaceAllUsesWith(Arg); - } - if (IsStart) { - Argument *OldArg = F->getArg(Idx); - if (OldArg->hasInRegAttr()) - Arg->addAttr(Attribute::InReg); - else - Arg->removeAttr(Attribute::AttrKind::InReg); - } + // Find the continuation state pointer, either returned by the malloc or + // given as an argument + Value *ContFrame = + getContFrame(FuncData.MallocCall, F, IsStart, InstsToRemove); + + // Create new empty function + F->eraseMetadata(FuncData.MD->getMetadataID()); + auto *NewFuncTy = FunctionType::get(Void, AllArgTypes, false); + Function *NewFunc = + CompilerUtils::cloneFunctionHeader(*F, NewFuncTy, ParamAttrs); + NewFunc->takeName(F); + NewFuncs.push_back({NewFunc, IsStart}); + + // Transfer code from old function to new function + llvm::moveFunctionBody(*F, *NewFunc); + + // Set arg names for new function + for (unsigned Idx = 0; Idx != NewFunc->getFunctionType()->params().size(); + ++Idx) { + Argument *Arg = NewFunc->getArg(Idx); + Value *OldVal = AllArgValues[Idx]; + if (OldVal) { + Arg->setName(OldVal->getName()); + OldVal->replaceAllUsesWith(Arg); + } + if (IsStart) { + Argument *OldArg = F->getArg(Idx); + if (OldArg->hasInRegAttr()) + Arg->addAttr(Attribute::InReg); + else + Arg->removeAttr(Attribute::AttrKind::InReg); } + } - // Handle the function entry - IRBuilder<> B(&*NewFunc->getEntryBlock().getFirstNonPHIOrDbgOrAlloca()); - if (IsStart) - FuncData.second.NewStart = NewFunc; - handleFunctionEntry(B, FuncData.second, NewFunc, IsEntry); - - // Handle the function body - // Use the global continuation state - ContFrame->replaceAllUsesWith(B.CreateBitOrPointerCast( - FuncData.second.NewContState, ContFrame->getType())); - - // Handle the function returns - for (auto &BB : make_early_inc_range(*NewFunc)) { - auto *I = BB.getTerminator(); - if (I->getOpcode() == Instruction::Ret) { - handleContinue(B, FuncData.second, I); - } else if (I->getOpcode() == Instruction::Unreachable) { - if (auto *Call = dyn_cast(--I->getIterator())) { - if (auto *Called = Call->getCalledFunction()) { - if (Called->getName() == "continuation.return") - handleReturn(B, FuncData.second, Call); - } + // Handle the function entry + B.SetInsertPoint(&*NewFunc->getEntryBlock().getFirstNonPHIOrDbgOrAlloca()); + if (IsStart) { + FuncData.NewStart = NewFunc; + ContMDTuple = + MDTuple::get(Context, {ValueAsMetadata::get(FuncData.NewStart)}); + } + handleFunctionEntry(FuncData, NewFunc, IsEntry); + + // Handle the function body + // Use the global continuation state + ContFrame->replaceAllUsesWith( + B.CreateBitOrPointerCast(FuncData.NewContState, ContFrame->getType())); + + // Handle the function returns + for (auto &BB : make_early_inc_range(*NewFunc)) { + auto *I = BB.getTerminator(); + if (I->getOpcode() == Instruction::Ret) { + handleContinue(FuncData, I); + } else if (I->getOpcode() == Instruction::Unreachable) { + if (auto *Call = dyn_cast(--I->getIterator())) { + if (auto *Called = Call->getCalledFunction()) { + if (Called->getName() == "continuation.return") + handleReturn(FuncData, Call); } } } + } - for (auto *I : InstsToRemove) - I->eraseFromParent(); + for (auto *I : InstsToRemove) + I->eraseFromParent(); - // Remove the old function - F->replaceAllUsesWith(ConstantExpr::getBitCast(NewFunc, F->getType())); - } - } + // Remove the old function + F->replaceAllUsesWith(ConstantExpr::getBitCast(NewFunc, F->getType())); + ToRemove.push_back(F); - // Remove the old functions and update metadata - for (auto &FuncData : ToProcess) { - if (FuncData.second.Functions.size() > 1) { - // Only for functions that were split - for (auto *F : FuncData.second.Functions) - F->eraseFromParent(); - - MDTuple *ContMDTuple = MDTuple::get( - Context, {ValueAsMetadata::get(FuncData.second.NewStart)}); - for (auto *F : FuncData.second.NewFunctions) { - F->setMetadata(DXILContHelper::MDContinuationName, ContMDTuple); - if (F != FuncData.second.NewStart) { - // For non-start functions, set (incoming) continuation registercount - // metadata by looking at the continue calls that reference this - // function. These continue calls both specify the number of their - // outgoing registers, and the number of incoming payload registers - // coming back into the resume function (i.e. us). - SmallVector Worklist(F->users()); - std::optional RegCount; - while (!Worklist.empty()) { - auto *U = Worklist.pop_back_val(); - if (auto *Const = dyn_cast(U)) { - Worklist.append(Const->user_begin(), Const->user_end()); - continue; - } - assert(isa(U) && - "User of a resume function should be a call to continue"); - auto *Inst = cast(U); - if (auto Count = - DXILContHelper::tryGetReturnedRegisterCount(Inst)) { - assert((!RegCount || *RegCount == *Count) && - "Got different returned registercounts in continues to " - "the same resume function"); - RegCount = *Count; -#ifdef NDEBUG - break; -#endif - } else { - LLVM_DEBUG(Inst->dump()); - report_fatal_error( - "Found a continue call without " - "continuation returned registercount metadata"); - } - } + // Update metadata + assert(ContMDTuple != nullptr); + NewFunc->setMetadata(DXILContHelper::MDContinuationName, ContMDTuple); + } - // Add metadata - DXILContHelper::setIncomingRegisterCount(F, RegCount.value()); - } - } + // Register count analysis needs to wait until all functions have been + // processed above, turning rets into continuation.[wait]continue calls. + for (auto [NewFunc, IsStart] : NewFuncs) { + if (!IsStart) { + uint32_t IncomingRegisterCount = getIncomingRegisterCount(NewFunc); + DXILContHelper::setIncomingRegisterCount(NewFunc, IncomingRegisterCount); } } - fixupDxilMetadata(*M); + for (auto *F : ToRemove) + F->eraseFromParent(); } -void LegacyCleanupContinuationsPass::handleFunctionEntry(IRBuilder<> &B, - ContinuationData &Data, - Function *F, - bool IsEntry) { +void LegacyCleanupContinuationsPassImpl::handleFunctionEntry( + ContinuationData &Data, Function *F, bool IsEntry) { bool IsStart = F == Data.NewStart; // Create alloca to keep the continuation state uint64_t ContStateNumI32s = divideCeil(Data.ContStateBytes, RegisterBytes); + uint64_t NeededStackSize = Data.getContStateStackBytes(); auto *ContStateTy = ArrayType::get(I32, ContStateNumI32s); Data.NewContState = B.CreateAlloca(ContStateTy, nullptr, "cont.state"); - uint64_t NeededStackSize = computeNeededStackSizeForRegisterBuffer( - ContStateNumI32s, ContinuationStateRegisterCount); if (IsStart) { // Add function metadata that stores how big the continuation state is in // bytes DXILContHelper::setContinuationStateByteCount(*F, Data.ContStateBytes); - - // Add intrinsic call to save the previous continuation state - if (!IsEntry && Data.ContStateBytes) - B.CreateCall(SaveContState); - if (NeededStackSize) { // Add to continuation stack size metadata DXILContHelper::addStackSize(F, NeededStackSize); } - } else { - // Read continuation state from global into local variable + } else if (NeededStackSize) { + // Obtain current CSP + auto *CspOffsetPtr = B.CreateCall(getContinuationStackOffset(M)); + auto *CspType = getContinuationStackOffsetType(M.getContext()); + auto *Offset = B.CreateLoad(CspType, CspOffsetPtr); + auto *Ptr = continuationStackOffsetToPtr( + B, Offset, *(GpurtLibrary ? GpurtLibrary : &M), CrossInliner); + + // Obtain ptr to continuation state on stack, + // and copy continuation state from global into local variable + Value *ContStateOnStack = + B.CreateGEP(B.getInt8Ty(), Ptr, B.getInt64(-NeededStackSize)); createCopy( B, Data.NewContState, - B.CreateBitOrPointerCast( - ContState, ContStateTy->getPointerTo(ContState->getAddressSpace())), + B.CreateBitOrPointerCast(ContStateOnStack, + ContStateTy->getPointerTo( + Ptr->getType()->getPointerAddressSpace())), ContStateTy); - // Deallocate continuation stack space if necessary - if (NeededStackSize) { - // Add barrier so that the csp is only decremented after the continuation - // state is read - auto *Csp = B.CreateCall( - getContinuationStackOffset(*B.GetInsertPoint()->getModule())); - B.CreateCall(RegisterBufferSetPointerBarrier, {ContState, Csp}); - - moveContinuationStackOffset(B, -NeededStackSize); - } + // Deallocate continuation stack space. + // The generated IR is partially redundant with the above, + // as the new CSP is just ContStateOnStack from above. + // However, we need to do the copy first and only then deallocate. + moveContinuationStackOffset(B, -NeededStackSize); } } @@ -461,9 +547,8 @@ void LegacyCleanupContinuationsPass::handleFunctionEntry(IRBuilder<> &B, /// unreachable /// /// Also handles cases where the token and resume function are behind a phi. -void LegacyCleanupContinuationsPass::handleContinue(IRBuilder<> &B, - ContinuationData &Data, - Instruction *Ret) { +void LegacyCleanupContinuationsPassImpl::handleContinue(ContinuationData &Data, + Instruction *Ret) { // Find the function call that generates the token LLVM_DEBUG(dbgs() << "Converting ret to continue: " << *Ret << "\nArgument: " << *Ret->getOperand(0) << "\n"); @@ -481,7 +566,7 @@ void LegacyCleanupContinuationsPass::handleContinue(IRBuilder<> &B, << "\n"); auto *Call = Entry.second.first; auto *ResumeFun = Entry.second.second; - handleSingleContinue(B, Data, Call, ResumeFun); + handleSingleContinue(Data, Call, ResumeFun); } if (BB->empty()) { @@ -491,8 +576,8 @@ void LegacyCleanupContinuationsPass::handleContinue(IRBuilder<> &B, } } -void LegacyCleanupContinuationsPass::handleSingleContinue( - IRBuilder<> &B, ContinuationData &Data, CallInst *Call, Value *ResumeFun) { +void LegacyCleanupContinuationsPassImpl::handleSingleContinue( + ContinuationData &Data, CallInst *Call, Value *ResumeFun) { // Pass resume address as argument B.SetInsertPoint(Call); auto *ReturnAddrInt = B.CreatePtrToInt(ResumeFun, I64); @@ -501,27 +586,25 @@ void LegacyCleanupContinuationsPass::handleSingleContinue( auto *CspFun = getContinuationStackOffset(*Call->getModule()); // Write local continuation state to stack and registers - uint64_t ContStateNumI32s = divideCeil(Data.ContStateBytes, RegisterBytes); - uint64_t NeededStackSize = computeNeededStackSizeForRegisterBuffer( - ContStateNumI32s, ContinuationStateRegisterCount); - + uint64_t NeededStackSize = Data.getContStateStackBytes(); if (NeededStackSize) { - // Allocate continuation stack space if necessary - moveContinuationStackOffset(B, NeededStackSize); - // Add barrier so that the csp is only incremented before the continuation - // state is written - auto *Csp = B.CreateCall(CspFun); - B.CreateCall(RegisterBufferSetPointerBarrier, {ContState, Csp}); + // Allocate continuation stack space + Value *ContStateOnStackOffset = + moveContinuationStackOffset(B, NeededStackSize).first; + auto *ContStateOnStackPtr = continuationStackOffsetToPtr( + B, ContStateOnStackOffset, *(GpurtLibrary ? GpurtLibrary : &M), + CrossInliner); + // Copy continuation state from local variable into global + auto *ContStateTy = Data.NewContState->getAllocatedType(); + createCopy( + B, + B.CreateBitOrPointerCast( + ContStateOnStackPtr, + ContStateTy->getPointerTo( + ContStateOnStackPtr->getType()->getPointerAddressSpace())), + Data.NewContState, ContStateTy); } - // Copy continuation state from local variable into global - auto *ContStateTy = Data.NewContState->getAllocatedType(); - createCopy( - B, - B.CreateBitOrPointerCast( - ContState, ContStateTy->getPointerTo(ContState->getAddressSpace())), - Data.NewContState, ContStateTy); - auto *Csp = B.CreateLoad(CpsType, B.CreateCall(CspFun)); bool IsWait = DXILContHelper::isWaitAwaitCall(*Call); @@ -562,9 +645,8 @@ void LegacyCleanupContinuationsPass::handleSingleContinue( /// call void @continuation.continue(i64 %returnaddr, i8 addrspace(21)* %csp, /// ) /// unreachable -void LegacyCleanupContinuationsPass::handleReturn(IRBuilder<> &B, - ContinuationData &Data, - CallInst *ContRet) { +void LegacyCleanupContinuationsPassImpl::handleReturn(ContinuationData &Data, + CallInst *ContRet) { LLVM_DEBUG(dbgs() << "Converting return to continue: " << *ContRet << "\n"); bool IsEntry = isa(ContRet->getArgOperand(0)); B.SetInsertPoint(ContRet); @@ -573,10 +655,6 @@ void LegacyCleanupContinuationsPass::handleReturn(IRBuilder<> &B, "Entry functions ignore the return value"); B.CreateCall(Complete); } else { - // Add intrinsic call to restore the previous continuation state - if (Data.ContStateBytes) - B.CreateCall(RestoreContState); - SmallVector Args(ContRet->args()); auto *CspType = getContinuationStackOffsetType(ContRet->getContext()); auto *CspFun = getContinuationStackOffset(*ContRet->getModule()); @@ -594,93 +672,69 @@ void LegacyCleanupContinuationsPass::handleReturn(IRBuilder<> &B, ContRet->eraseFromParent(); } -llvm::PreservedAnalyses LegacyCleanupContinuationsPass::run( - llvm::Module &Mod, llvm::ModuleAnalysisManager &AnalysisManager) { - LLVM_DEBUG(dbgs() << "Run the cleanup-continuations pass\n"); - AnalysisManager.getResult(Mod); +LegacyCleanupContinuationsPassImpl::LegacyCleanupContinuationsPassImpl( + llvm::Module &Mod, llvm::Module *GpurtLibrary, + llvm::ModuleAnalysisManager &AnalysisManager) + : M{Mod}, Context{M.getContext()}, B{Context}, GpurtLibrary{GpurtLibrary} { + AnalysisManager.getResult(M); + ContMalloc = M.getFunction("continuation.malloc"); + ContFree = M.getFunction("continuation.free"); +} - M = &Mod; - ToProcess.clear(); - MaxContStateBytes = 0; - ContMalloc = Mod.getFunction("continuation.malloc"); - ContFree = Mod.getFunction("continuation.free"); +llvm::PreservedAnalyses LegacyCleanupContinuationsPassImpl::run() { + bool Changed = false; // Map the entry function of a continuation to the analysis result - for (auto &F : Mod.functions()) { + for (auto &F : M.functions()) { if (F.empty()) continue; - if (auto *MD = F.getMetadata(DXILContHelper::MDContinuationName)) + + if (auto *MD = F.getMetadata(DXILContHelper::MDContinuationName)) { analyzeContinuation(F, MD); + } else if (auto Stage = lgc::rt::getLgcRtShaderStage(&F); + Stage && *Stage == lgc::rt::RayTracingShaderStage::Traversal) { + Changed = true; + // Add !continuation metadata to Traversal after coroutine passes. + // The traversal loop is written as like the coroutine passes were applied + // manually. + MDTuple *ContMDTuple = MDTuple::get(Context, {ValueAsMetadata::get(&F)}); + F.setMetadata(DXILContHelper::MDContinuationName, ContMDTuple); + } } // Check if the continuation state is used in any function part for (auto &FuncData : ToProcess) { - if (!FuncData.second.MallocCall) { - for (auto *F : FuncData.second.Functions) { - bool IsStart = - (F == FuncData.first); // If this is the continuation start - Value *ContFrame; - if (IsStart) - ContFrame = F->getArg(F->arg_size() - 1); - else - ContFrame = F->getArg(0); - // If there are uses, we need to assume a size of - // MinimumContinuationStateBytes, because for all sizes up to this size - // coroutine passes will not emit a malloc that we can use to determine - // the exact size. If however the frame pointer is not used in any of - // the continuation functions, it's safe to assume an empty continuation - // state. - if (!ContFrame->user_empty()) { - FuncData.second.ContStateBytes = MinimumContinuationStateBytes; - if (MinimumContinuationStateBytes > MaxContStateBytes) - MaxContStateBytes = MinimumContinuationStateBytes; - } - } - } + finalizeContinuationData(*FuncData.first, FuncData.second); + MaxContStateBytes = + std::max(MaxContStateBytes, FuncData.second.ContStateBytes); } + Changed |= !ToProcess.empty(); + if (!ToProcess.empty()) { - auto &Context = Mod.getContext(); I32 = Type::getInt32Ty(Context); I64 = Type::getInt64Ty(Context); - SaveContState = getContinuationSaveContinuationState(Mod); - RestoreContState = getContinuationRestoreContinuationState(Mod); - Continue = getContinuationContinue(Mod); - WaitContinue = getContinuationWaitContinue(Mod); - Complete = getContinuationComplete(Mod); - - // Add global - // Size is the maximum of all continuations, but at least the register size - uint32_t ContStateSize = std::max( - MaxContStateBytes, ContinuationStateRegisterCount * RegisterBytes); - auto *ContStateTy = - ArrayType::get(I32, divideCeil(ContStateSize, RegisterBytes)); - ContState = cast(Mod.getOrInsertGlobal( - DXILContHelper::GlobalContStateName, ContStateTy, [&] { - return new GlobalVariable(Mod, ContStateTy, false, - GlobalVariable::ExternalLinkage, nullptr, - DXILContHelper::GlobalContStateName, - nullptr, GlobalVariable::NotThreadLocal); - })); - - RegisterBufferSetPointerBarrier = getRegisterBufferSetPointerBarrier(Mod); - - // Add registerbuffer metadata to split accesses at into i32s and spill to - // memory if necessary - std::optional StackAddrspace = - DXILContHelper::tryGetStackAddrspace(*M); - if (!StackAddrspace) - report_fatal_error("Missing stack addrspace metadata!"); - RegisterBufferMD RMD; - RMD.RegisterCount = ContinuationStateRegisterCount; - RMD.Addrspace = static_cast(*StackAddrspace); - auto *MD = createRegisterBufferMetadata(Context, RMD); - ContState->addMetadata("registerbuffer", *MD); - - processContinuations(); + Continue = getContinuationContinue(M); + WaitContinue = getContinuationWaitContinue(M); + Complete = getContinuationComplete(M); + + for (auto &FuncData : ToProcess) { + processContinuation(FuncData.first, FuncData.second); + } + + fixupDxilMetadata(M); } - if (!ToProcess.empty()) + if (Changed) return PreservedAnalyses::none(); return PreservedAnalyses::all(); } + +} // namespace + +llvm::PreservedAnalyses LegacyCleanupContinuationsPass::run( + llvm::Module &Mod, llvm::ModuleAnalysisManager &AnalysisManager) { + LLVM_DEBUG(dbgs() << "Run the cleanup-continuations pass\n"); + LegacyCleanupContinuationsPassImpl Impl(Mod, GpurtLibrary, AnalysisManager); + return Impl.run(); +} diff --git a/shared/continuations/lib/LowerAwait.cpp b/shared/continuations/lib/LowerAwait.cpp index 3080f69a9c..41d1de649e 100644 --- a/shared/continuations/lib/LowerAwait.cpp +++ b/shared/continuations/lib/LowerAwait.cpp @@ -34,6 +34,7 @@ // //===----------------------------------------------------------------------===// +#include "compilerutils/CompilerUtils.h" #include "continuations/Continuations.h" #include "continuations/ContinuationsDialect.h" #include "lgc/LgcCpsDialect.h" @@ -52,32 +53,6 @@ using namespace llvm; #define DEBUG_TYPE "lower-await" -Function *llvm::getContinuationSaveContinuationState(Module &M) { - auto *Name = "continuation.save.continuation_state"; - if (auto *F = M.getFunction(Name)) - return F; - auto &C = M.getContext(); - auto *Void = Type::getVoidTy(C); - AttributeList AL = AttributeList::get( - C, AttributeList::FunctionIndex, - {Attribute::NoFree, Attribute::NoRecurse, Attribute::NoSync, - Attribute::NoUnwind, Attribute::WillReturn}); - return cast(M.getOrInsertFunction(Name, AL, Void).getCallee()); -} - -Function *llvm::getContinuationRestoreContinuationState(Module &M) { - auto *Name = "continuation.restore.continuation_state"; - if (auto *F = M.getFunction(Name)) - return F; - auto &C = M.getContext(); - auto *Void = Type::getVoidTy(C); - AttributeList AL = AttributeList::get( - C, AttributeList::FunctionIndex, - {Attribute::NoFree, Attribute::NoRecurse, Attribute::NoSync, - Attribute::NoUnwind, Attribute::WillReturn}); - return cast(M.getOrInsertFunction(Name, AL, Void).getCallee()); -} - Function *llvm::getContinuationContinue(Module &M) { auto *Name = "continuation.continue"; if (auto *F = M.getFunction(Name)) @@ -211,7 +186,8 @@ static void processContinuations( // Create new empty function auto *NewFuncTy = FunctionType::get(NewRetTy, AllArgTypes, false); - Function *NewFunc = cloneFunctionHeader(*F, NewFuncTy, {}); + Function *NewFunc = CompilerUtils::cloneFunctionHeader( + *F, NewFuncTy, ArrayRef{}); NewFunc->takeName(F); // Transfer code from old function to new function diff --git a/shared/continuations/lib/LowerRaytracingPipeline.cpp b/shared/continuations/lib/LowerRaytracingPipeline.cpp index 12eb462fb5..9354dacfb5 100644 --- a/shared/continuations/lib/LowerRaytracingPipeline.cpp +++ b/shared/continuations/lib/LowerRaytracingPipeline.cpp @@ -39,7 +39,7 @@ // //===----------------------------------------------------------------------===// -#include "continuations/LowerRaytracingPipeline.h" +#include "compilerutils/CompilerUtils.h" #include "continuations/Continuations.h" #include "continuations/ContinuationsDialect.h" #include "continuations/ContinuationsUtil.h" @@ -51,12 +51,15 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/IR/Attributes.h" +#include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" +#include "llvm/IR/Value.h" +#include "llvm/IR/Verifier.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" @@ -66,6 +69,8 @@ #include #include #include +#include +#include #include #define DEBUG_TYPE "lower-raytracing-pipeline" @@ -213,7 +218,244 @@ struct PayloadCopyHelper { } }; -} // anonymous namespace +enum class ContinuationCallType { + Traversal, + CallShader, + AnyHit, +}; + +class ModuleMetadataState final { +public: + ModuleMetadataState(llvm::Module &Module); + ModuleMetadataState(const ModuleMetadataState &) = delete; + ModuleMetadataState(ModuleMetadataState &&) = default; + + uint32_t getMaxPayloadRegisterCount() const { + return MaxPayloadRegisterCount; + } + + uint32_t getMinPayloadRegisterCount() const { + return MinPayloadRegisterCount; + } + + ContStackAddrspace getContStackAddrspace() const { return StackAddrspace; } + + void updateModuleMetadata() const; + +private: + Module &Mod; + /// MaxPayloadRegisterCount is initialized from metadata. If there is none, + /// use this default instead: + static constexpr uint32_t DefaultPayloadRegisterCount = 30; + /// Maximum allowed number of registers to be used for the payload. + uint32_t MaxPayloadRegisterCount = 0; + /// Minimum required number of payload registers. + uint32_t MinPayloadRegisterCount = 0; + /// The address space used for the continuations stack. + /// Either stack or global memory. + ContStackAddrspace StackAddrspace = DXILContHelper::DefaultStackAddrspace; +}; + +class CpsMutator final { +public: + explicit CpsMutator(Module &Mod) + : Mod{Mod}, IsModuleInCpsMode{DXILContHelper::isLgcCpsModule(Mod)}, + Builder{std::make_unique(Mod.getContext())} {} + + Value *insertCpsAwait(Type *ReturnTy, Value *ShaderAddr, Instruction *Call, + ArrayRef Args, ContinuationCallType CallType, + lgc::cps::CpsShaderStage ShaderStage); + + bool shouldRun() const { return IsModuleInCpsMode; } + +private: + Module &Mod; + bool IsModuleInCpsMode = false; + std::unique_ptr Builder; +}; + +class LowerRaytracingPipelinePassImpl final { +public: + LowerRaytracingPipelinePassImpl(Module &M, Module &GpurtLibrary); + bool run(); + +private: + struct FunctionConfig { + // Maximum allowed size of hit attributes to be used in a TraceRay together + // with this function, even if this function does not touch hit attributes + // (e.g. a Miss shader). + uint32_t MaxHitAttributeBytes = 0; + + bool operator==(const FunctionConfig &Other) const { + return MaxHitAttributeBytes == Other.MaxHitAttributeBytes; + } + }; + + struct FunctionData { + DXILShaderKind Kind = DXILShaderKind::Invalid; + SmallVector TraceRayCalls; + SmallVector ReportHitCalls; + SmallVector CallShaderCalls; + /// Calls to hlsl intrinsics that cannot be rematerialized + SmallVector IntrinsicCalls; + SmallVector ShaderIndexCalls; + + /// Pointer to the alloca'd system data object in this function + AllocaInst *SystemData = nullptr; + StructType *SystemDataTy = nullptr; + Type *ReturnTy = nullptr; + /// Maximum number of I32s required to store the outgoing payload in all + /// CallShader or TraceRay (maximum over all TraceRay formats) calls + uint32_t MaxOutgoingPayloadI32s = 0; + /// Size of the CPS stack allocation used for spilled parts of the payload. + /// This size is large enough for all used outgoing payload types. + int32_t PayloadSpillSize = 0; + /// Type of the incoming payload + Type *IncomingPayload = nullptr; + FunctionConfig FuncConfig = {}; + /// Serialization info for the incoming payload, if there is one. + /// Also applies to the outgoing payload in that case. + PAQSerializationInfoBase *IncomingPayloadSerializationInfo = nullptr; + /// hit attributes type, incoming for AnyHit and ClosestHit, outgoing for + /// Intersection + Type *HitAttributes = nullptr; + }; + + /// Needed data for handling the end of a function + struct FunctionEndData { + Instruction *Terminator = nullptr; + const PAQSerializationLayout *OutgoingSerializationLayout = nullptr; + SmallVector SavedRegisterValues; + Value *NewPayload = nullptr; + std::optional ShaderStage; + Value *HitAttrsAlloca = nullptr; + Value *OrigHitAttrsAlloca = nullptr; + Type *NewRetTy = nullptr; + }; + + static DXILShaderKind callTypeToShaderKind(ContinuationCallType CallType); + + void replaceCall(FunctionData &Data, CallInst *Call, Function *Func, + ContinuationCallType CallType); + void handleRestoreSystemData(CallInst *Call); + void replaceContinuationCall(ContinuationCallType CallType, CallInst *Call, + const FunctionData &Data, Value *PayloadOrAttrs, + Type *PayloadOrAttrsTy); + void replaceReportHitCall(FunctionData &Data, CallInst *Call); + + void handleReportHit(FunctionData &Data, Function &F); + + void replaceShaderIndexCall(FunctionData &Data, CallInst *Call); + + void handleGetFuncAddr(Function &Func); + void handleGetShaderKind(Function &Func); + void handleGetCurrentFuncAddr(Function &Func); + + void handleAmdInternalFunc(Function &Func); + + void splitRestoreBB(); + + void handleUnrematerializableCandidates(); + + void collectDriverFunctions(); + + // Copy the payload content between global payload and local payload. + // Excludes the stack pointer or hit attributes which may also reside in + // payload storage. If Stage is not set, all fields in SerializationInfo are + // copied. Used for CallShader accesses which are not PAQ qualified and do not + // have PAQShaderStage values. + // If CopiedNodes is set, nodes contained will not be copied, and all copied + // nodes are added to it. + void copyPayload(Type &PayloadTy, Value *LocalPayload, + std::optional Stage, + PAQAccessKind GlobalAccessKind, + const PAQSerializationLayout &Layout, + SmallDenseSet *CopiedNodes = nullptr); + + // Special handling for case of copying the result payload of a traceray call + // back to the local payload of the caller. + // This is needed to implement the ClosestHitOut/MissOut optimization. + // We first perform a copy using the ClosestHitOut layout, and then perform an + // additional copy using the MissOut layout, skipping any fields already + // copied (i.e. only copying write(miss) : read(caller) fields). + void copyTraceRayPayloadIncomingToCaller( + const PAQTraceRaySerializationInfo &PAQSerializationInfo, + Value *LocalPayload); + + // Caller-save payload registers before CallShader() or TraceRay(), + // which can override payload registers. A register needs to be saved + // if it is live in OutgoingLayout, and not written in OutgoingLayout. + // This includes the payload memory pointer if present. + // SavedRegisters maps indices of payload registers to their saved values. + void savePayloadRegistersBeforeRecursion( + DXILShaderKind Kind, const PAQSerializationLayout &IncomingLayout, + const PAQSerializationLayout &OutgoingLayout, + SmallVectorImpl &SavedRegisterValues); + + // Restore previously saved registers. + void restorePayloadRegistersAfterRecursion( + const SmallVectorImpl &SavedRegisterValues); + + void createPayloadGlobal(); + + void setTraversalRegisterCountMetadata(); + + void copyHitAttributes(FunctionData &Data, Value *SystemData, + Type *SystemDataTy, Value *LocalHitAttributes, + bool GlobalToLocal, + const PAQSerializationLayout *Layout); + void processContinuations(); + void processFunctionEntry(Function *F, FunctionData &Data); + void processFunctionEnd(FunctionData &Data, FunctionEndData &EData); + void processFunction(Function *F, FunctionData &FuncData); + + void collectProcessableFunctions(); + + void handleDriverFuncAssertions(); + + constexpr static uint32_t ArgContState = 0; + constexpr static uint32_t ArgReturnAddr = 1; + constexpr static uint32_t ArgShaderIndex = 2; + [[maybe_unused]] constexpr static uint32_t ArgSystemData = 3; + [[maybe_unused]] constexpr static uint32_t ArgHitAttributes = 4; + + MapVector ToProcess; + Module *Mod; + Module *GpurtLibrary; + LLVMContext *Context; + const DataLayout *DL; + llvm_dialects::Builder Builder; + ModuleMetadataState MetadataState; + CpsMutator Mutator; + PAQSerializationInfoManager PAQManager; + CompilerUtils::CrossModuleInliner CrossInliner; + Type *I32; + Type *TokenTy; + /// System data type passed to Traversal + Type *TraversalDataTy; + /// System data type passed to ClosestHit and Miss + Type *HitMissDataTy; + GlobalVariable *Payload; + + // Function definitions and declarations from HLSL + // Driver implementation that returns if AcceptHitAndEndSearch was called + Function *IsEndSearch; + // Driver implementations to set and get the triangle hit attributes from + // system data + Function *GetTriangleHitAttributes; + Function *SetTriangleHitAttributes; + Function *GetLocalRootIndex; + Function *SetLocalRootIndex; + Function *SetupRayGen; + Function *TraceRay; + Function *CallShader; + Function *ReportHit; + Function *AcceptHit; + + Function *RegisterBufferSetPointerBarrier; +}; + +} // namespace constexpr unsigned ModuleMetadataState::DefaultPayloadRegisterCount; @@ -313,45 +555,10 @@ Function *llvm::getSetLocalRootIndex(Module &M) { } // Set maximum continuation stack size metadata -static void setStacksizeMetadata(Function &F, uint64_t PayloadI32s, - uint64_t PayloadRegisterCount) { - uint64_t NeededStackSize = computeNeededStackSizeForRegisterBuffer( - PayloadI32s, PayloadRegisterCount); - if (NeededStackSize) { - uint64_t CurStackSize = DXILContHelper::tryGetStackSize(&F).value_or(0); - if (NeededStackSize > CurStackSize) - DXILContHelper::setStackSize(&F, NeededStackSize); - } -} - -bool llvm::isRematerializableLgcRtOp(CallInst &CInst, - std::optional Kind) { - Function *Callee = CInst.getCalledFunction(); - if (!DialectUtils::isLgcRtOp(Callee)) - return false; - - // Always rematerialize - static const OpSet RematerializableDialectOps = - OpSet::get(); - if (RematerializableDialectOps.contains(*Callee)) - return true; - - // Rematerialize for Intersection that can only call ReportHit, which keeps - // the largest system data struct. These cannot be rematerialized in - // ClosestHit, because if ClosestHit calls TraceRay or CallShader, that - // information is lost from the system data struct. Also exclude rayTCurrent - // because ReportHit calls can change that. - if (!Kind || *Kind == DXILShaderKind::Intersection) { - static const OpSet RematerializableIntersectionDialectOps = - OpSet::get(); - if (RematerializableIntersectionDialectOps.contains(*Callee)) - return true; - } - - return false; +static void setStacksizeMetadata(Function &F, uint64_t NeededStackSize) { + uint64_t CurStackSize = DXILContHelper::tryGetStackSize(&F).value_or(0); + if (NeededStackSize > CurStackSize) + DXILContHelper::setStackSize(&F, NeededStackSize); } // Create an ExtractElement instruction for each index of a FixedVector @Vector @@ -403,14 +610,17 @@ DXILShaderKind LowerRaytracingPipelinePassImpl::callTypeToShaderKind( } /// Clone a function and replace a call with a call to the cloned function -CallInst *LowerRaytracingPipelinePassImpl::replaceCall( - IRBuilder<> &B, FunctionData &Data, CallInst *Call, Function *Func, - StringRef NewName, ContinuationCallType CallType) { - B.SetInsertPoint(Call); +void LowerRaytracingPipelinePassImpl::replaceCall( + FunctionData &Data, CallInst *Call, Function *Func, + ContinuationCallType CallType) { + Builder.SetInsertPoint(Call); + auto *AfterCall = &*++Builder.GetInsertPoint(); auto *SystemDataTy = getFuncArgPtrElementType(Func, 0); + Value *PayloadOrAttrs = nullptr; SmallVector Arguments; - Arguments.push_back(getDXILSystemData(B, Data.SystemData, Data.SystemDataTy, + Arguments.push_back(getDXILSystemData(Builder, Data.SystemData, + Data.SystemDataTy, cast(SystemDataTy))); // Construct the new argument list for the driver-side call from a lgc.rt @@ -422,15 +632,16 @@ CallInst *LowerRaytracingPipelinePassImpl::replaceCall( case ContinuationCallType::Traversal: { // Generally exclude the last (PAQ) argument. const unsigned ArgCount = Call->arg_size(); - for (unsigned CallI = 0; CallI < ArgCount - 1; ++CallI) { + for (unsigned CallI = 0; CallI < ArgCount - 2; ++CallI) { // For trace.ray calls, we need to flatten all vectors in the argument - // list. Skip the payload for now. + // list. Value *Arg = Call->getArgOperand(CallI); - if (CallI < ArgCount - 2 && flattenVectorArgument(B, Arg, Arguments)) + if (flattenVectorArgument(Builder, Arg, Arguments)) continue; Arguments.push_back(Arg); } + PayloadOrAttrs = Call->getArgOperand(Call->arg_size() - 2); break; } @@ -440,90 +651,64 @@ CallInst *LowerRaytracingPipelinePassImpl::replaceCall( // For the report.hit operation, we remove the PAQ size attribute since it // is included in the name. For the call.callable.shader operation, we // remove the PAQ size attribute as well since it is not supported. - Arguments.append(Call->arg_begin(), Call->arg_end() - 1); + Arguments.append(Call->arg_begin(), Call->arg_end() - 2); + PayloadOrAttrs = Call->getArgOperand(Call->arg_size() - 2); break; } - Function *SpecializedFunction = Mod->getFunction(NewName); - AwaitFunctionData AwaitData; - AwaitData.CallType = CallType; - AwaitData.FuncConfig = Data.FuncConfig; - AwaitData.CallerKind = Data.Kind; - if (!SpecializedFunction) { - // Copy function, modify argument types - SmallVector ArgTys; - ArgTys.reserve(Func->getFunctionType()->params().size() + 1); - - // Add system data argument - ArgTys.push_back(SystemDataTy->getPointerTo()); - - // Skip intrinsic id argument - ArgTys.append(Func->getFunctionType()->params().begin() + 1, - Func->getFunctionType()->params().end()); - - // Add payload argument - ArgTys.push_back(Call->getArgOperand(Call->arg_size() - 2)->getType()); - - SpecializedFunction = cloneFunctionHeader( - *Func, FunctionType::get(Func->getReturnType(), ArgTys, false), {}); - SpecializedFunction->setName(NewName); - - assert(PayloadOrAttrTypesForSpecializedFunctions.count( - SpecializedFunction) == 0); - // Store payload or hit attribute type for later. Despite the name, payload - // metadata also gives hit attribute types for ReportHit. - PayloadOrAttrTypesForSpecializedFunctions[SpecializedFunction] = - DXILContHelper::getPayloadTypeFromMetadata(*Call); - - assert(!AwaitsToProcess.count(SpecializedFunction) && - "Unexpected existing await data entry!"); - AwaitsToProcess[SpecializedFunction] = AwaitData; - - ValueToValueMapTy VMap; - // Set arg names for new function - for (unsigned Idx = 0; Idx != Func->getFunctionType()->params().size(); - ++Idx) { - Argument *Arg = SpecializedFunction->getArg(Idx); - Argument *OldArg = Func->getArg(Idx); - VMap[OldArg] = Arg; - Arg->setName(OldArg->getName()); - if (OldArg->hasInRegAttr()) - Arg->addAttr(Attribute::InReg); - else - Arg->removeAttr(Attribute::AttrKind::InReg); - } - - // Copy code - SmallVector Returns; - CloneFunctionInto(SpecializedFunction, Func, VMap, - CloneFunctionChangeType::LocalChangesOnly, Returns); - - // Do not propagate type metadata to the cloned function. It would be - // incorrect, because arguments differ, and we should no longer need it. - SpecializedFunction->setMetadata(DXILContHelper::MDTypesName, nullptr); - - if (CallType == ContinuationCallType::AnyHit) - handleReportHit(Data, *SpecializedFunction); - } else { -#ifndef NDEBUG - // Check that the already registered await data is consistent - auto AwaitDataIt = AwaitsToProcess.find(SpecializedFunction); - assert(AwaitDataIt != AwaitsToProcess.end() && "Missing await data!"); - assert(AwaitDataIt->second == AwaitData && "Inconsistent await data!"); -#endif - } + // Get payload argument + Type *PayloadOrAttrsTy = DXILContHelper::getPayloadTypeFromMetadata(*Call); + auto *NewCall = Builder.CreateCall(Func, Arguments); - auto *NewResult = B.CreateCall(SpecializedFunction, Arguments); if (!Call->getType()->isVoidTy()) - Call->replaceAllUsesWith(NewResult); + Call->replaceAllUsesWith(NewCall); Call->eraseFromParent(); + auto NewBlocks = CrossInliner.inlineCall(*NewCall); + + // Find special calls. Collect before replacing because replacing them inlines + // functions and changes basic blocks. + SmallVector AwaitCalls; + SmallVector AcceptHitAttrsCalls; + for (auto &BB : NewBlocks) { + for (auto &I : BB) { + if (auto *CI = dyn_cast(&I)) { + auto *Callee = CI->getCalledFunction(); + if (!Callee) + continue; + auto FuncName = Callee->getName(); + if (FuncName.starts_with("_AmdAwait") || + FuncName.starts_with("_AmdWaitAwait")) { + AwaitCalls.push_back(CI); + } else if (FuncName.starts_with("_AmdAcceptHitAttributes")) { + AcceptHitAttrsCalls.push_back(CI); + } + } + } + } + + for (auto *CI : AwaitCalls) { + Builder.SetInsertPoint(CI); + replaceContinuationCall(CallType, CI, Data, PayloadOrAttrs, + PayloadOrAttrsTy); + } - B.SetInsertPoint(&*++NewResult->getIterator()); - return NewResult; + for (auto *CI : AcceptHitAttrsCalls) { + // Commit hit attributes + Builder.SetInsertPoint(CI); + assert(TraversalDataTy != 0 && "Missing traversal system data!"); + copyHitAttributes(Data, CI->getArgOperand(0), TraversalDataTy, + PayloadOrAttrs, false, nullptr); + // Make sure that we store the hit attributes into the correct system + // data (just in case dxc copied them around). + assert(CI->getArgOperand(0) == Arguments[0] && + "AcceptHitAttributes does not take the correct system data as " + "argument!"); + CI->eraseFromParent(); + } + Builder.SetInsertPoint(AfterCall); } -void LowerRaytracingPipelinePassImpl::handleRestoreSystemData(IRBuilder<> &B, - CallInst *Call) { +void LowerRaytracingPipelinePassImpl::handleRestoreSystemData(CallInst *Call) { // Store system data auto *SystemDataTy = cast(getFuncArgPtrElementType(Call->getCalledFunction(), 0)); @@ -533,95 +718,52 @@ void LowerRaytracingPipelinePassImpl::handleRestoreSystemData(IRBuilder<> &B, assert(GetLocalRootIndex && "Could not find GetLocalRootIndex function"); auto *LocalIndexSystemDataTy = cast(getFuncArgPtrElementType(GetLocalRootIndex, 0)); - auto *LocalIndexSystemData = - getDXILSystemData(B, SystemData, SystemDataTy, LocalIndexSystemDataTy); - auto *LocalIndex = B.CreateCall(GetLocalRootIndex, LocalIndexSystemData); + auto *LocalIndexSystemData = getDXILSystemData( + Builder, SystemData, SystemDataTy, LocalIndexSystemDataTy); + auto *LocalIndex = + CrossInliner.inlineCall(Builder, GetLocalRootIndex, LocalIndexSystemData) + .returnValue; LocalIndex->setName("local.root.index"); - B.CreateCall(SetLocalRootIndex, LocalIndex); -} - -/// Replace a call to lgc.rt.trace.ray with a call to the driver TraceRay -/// implementation. -void LowerRaytracingPipelinePassImpl::replaceTraceRayCall(IRBuilder<> &B, - FunctionData &Data, - CallInst *Call) { - assert(TraceRay && "TraceRay not found"); - std::string TraceRayName = TraceRay->getName().str(); - TraceRayName += "."; - auto *PayloadType = DXILContHelper::getPayloadTypeFromMetadata(*Call); - - if (PayloadType->getStructName().empty()) - cast(PayloadType)->setName("struct.Payload"); - TraceRayName += PayloadType->getStructName(); - TraceRayName += Data.FuncConfig.getFunctionSuffix(); - - replaceCall(B, Data, Call, TraceRay, TraceRayName, - ContinuationCallType::Traversal); -} - -/// Replace a call to lgc.rt.call.callable.shader with a call to the driver -/// CallShader implementation. -void LowerRaytracingPipelinePassImpl::replaceCallShaderCall(IRBuilder<> &B, - FunctionData &Data, - CallInst *Call) { - assert(CallShader && "CallShader not found"); - std::string CallShaderName = CallShader->getName().str(); - CallShaderName += "."; - auto *PayloadType = DXILContHelper::getPayloadTypeFromMetadata(*Call); - - if (PayloadType->getStructName().empty()) - cast(PayloadType)->setName("struct.Payload"); - CallShaderName += PayloadType->getStructName(); - replaceCall(B, Data, Call, CallShader, CallShaderName, - ContinuationCallType::CallShader); + Builder.CreateCall(SetLocalRootIndex, LocalIndex); } /// Replace a call to lgc.rt.report.hit with a call to the driver /// implementation. -void LowerRaytracingPipelinePassImpl::replaceReportHitCall( - llvm_dialects::Builder &B, FunctionData &Data, CallInst *Call) { +void LowerRaytracingPipelinePassImpl::replaceReportHitCall(FunctionData &Data, + CallInst *Call) { assert(ReportHit && "ReportHit not found"); Function *F = Call->getFunction(); - std::string ReportHitName = ReportHit->getName().str(); - ReportHitName += "."; - - auto *HitAttrTy = DXILContHelper::getPayloadTypeFromMetadata(*Call); - if (HitAttrTy->getStructName().empty()) - cast(HitAttrTy)->setName("struct.HitAttr"); - ReportHitName += HitAttrTy->getStructName(); - - replaceCall(B, Data, Call, ReportHit, ReportHitName, - ContinuationCallType::AnyHit); + replaceCall(Data, Call, ReportHit, ContinuationCallType::AnyHit); // Check if the search ended and return from Intersection if this is the case assert(IsEndSearch && "IsEndSearch not found"); auto *SystemDataTy = getFuncArgPtrElementType(IsEndSearch, 0); - auto *SystemData = - getDXILSystemData(B, Data.SystemData, Data.SystemDataTy, SystemDataTy); - auto *IsEnd = B.CreateCall(IsEndSearch, SystemData); + auto *SystemData = getDXILSystemData(Builder, Data.SystemData, + Data.SystemDataTy, SystemDataTy); + auto *IsEnd = + CrossInliner.inlineCall(Builder, IsEndSearch, SystemData).returnValue; Instruction *Then = - SplitBlockAndInsertIfThen(IsEnd, &*B.GetInsertPoint(), true); - B.SetInsertPoint(Then); - SystemData = getDXILSystemData(B, Data.SystemData, Data.SystemDataTy, + SplitBlockAndInsertIfThen(IsEnd, &*Builder.GetInsertPoint(), true); + Builder.SetInsertPoint(Then); + SystemData = getDXILSystemData(Builder, Data.SystemData, Data.SystemDataTy, cast(Data.ReturnTy)); - Value *RetSystemData = B.CreateLoad(Data.ReturnTy, SystemData); + Value *RetSystemData = Builder.CreateLoad(Data.ReturnTy, SystemData); if (Mutator.shouldRun()) { uint32_t CpsRetLevel = getPotentialCpsReturnLevels( convertShaderKindToCpsShaderStage(Data.Kind)); - B.create(F->getArg(ArgReturnAddr), CpsRetLevel, - PoisonValue::get(StructType::get(B.getContext())), - RetSystemData); - B.CreateUnreachable(); + Builder.create( + F->getArg(ArgReturnAddr), CpsRetLevel, + PoisonValue::get(StructType::get(Builder.getContext())), RetSystemData); + Builder.CreateUnreachable(); } else { - auto *Ret = B.CreateRet(RetSystemData); + auto *Ret = Builder.CreateRet(RetSystemData); // Assume worst-case payload size for Intersection. See the note on the // incoming payload size. DXILContHelper::setOutgoingRegisterCount( - Ret, ContinuationStateRegisterCount + - MetadataState.getMaxPayloadRegisterCount()); + Ret, MetadataState.getMaxPayloadRegisterCount()); } // Remove trailing unreachable @@ -634,19 +776,9 @@ void LowerRaytracingPipelinePassImpl::replaceReportHitCall( /// - Call given address and pass generated token into an await call /// - Read payload from global void LowerRaytracingPipelinePassImpl::replaceContinuationCall( - IRBuilder<> &B, ContinuationCallType CallType, CallInst *Call, - const FunctionConfig &FuncConfig, DXILShaderKind CallerKind) { - B.SetInsertPoint(Call); - auto *F = Call->getFunction(); - Argument *PassedPayload = nullptr; - Type *PayloadTy = nullptr; - - // Payload passed to TraceRay or CallShader - if (CallType != ContinuationCallType::AnyHit) { - // Payload is unchanged by Intersection and passed implicitly - PassedPayload = F->getArg(F->arg_size() - 1); - PayloadTy = PayloadOrAttrTypesForSpecializedFunctions.at(F); - } + ContinuationCallType CallType, CallInst *Call, const FunctionData &Data, + Value *PayloadOrAttrs, Type *PayloadOrAttrsTy) { + Builder.SetInsertPoint(Call); const PAQSerializationLayout *OutgoingSerializationLayout = nullptr; const PAQSerializationInfoBase *OutgoingSerializationInfo = nullptr; @@ -654,10 +786,11 @@ void LowerRaytracingPipelinePassImpl::replaceContinuationCall( // current continuation call. std::optional ReturnedRegisterCount; std::optional ShaderStage; - if (PayloadTy) { + if (CallType != ContinuationCallType::AnyHit) { // Specify hit attribute size also in case it is used for CallShader. // It is ignored by the implementation in that case. - PAQPayloadConfig PAQConfig = {PayloadTy, FuncConfig.MaxHitAttributeBytes}; + PAQPayloadConfig PAQConfig = {PayloadOrAttrsTy, + Data.FuncConfig.MaxHitAttributeBytes}; if (CallType == ContinuationCallType::Traversal) { const auto *TraceRayInfo = &PAQManager.getOrCreateTraceRaySerializationInfo(PAQConfig); @@ -694,21 +827,21 @@ void LowerRaytracingPipelinePassImpl::replaceContinuationCall( // For intersection, assume maximum possible number of payload registers. ReturnedRegisterCount = MetadataState.getMaxPayloadRegisterCount(); } - // Allocate space for new payload if necessary - int64_t NeededStackSize = 0; if (OutgoingSerializationLayout) { - // Allocate space for the maximum occurring storage for the payload type - NeededStackSize = computeNeededStackSizeForRegisterBuffer( - OutgoingSerializationInfo->MaxStorageI32s, - MetadataState.getMaxPayloadRegisterCount()); - assert( - (NeededStackSize != 0) == - (OutgoingSerializationLayout->PayloadMemPointerNode != nullptr) && - "Inconsistent payload stack size!"); - if (NeededStackSize) { + // Set up the payload spill pointer if necessary + if (OutgoingSerializationLayout->PayloadMemPointerNode) { + // If we have a mem pointer, then we need to allocate stack storage + // The reverse does not hold, as a different payload type in the same + // shader could require the allocation. + assert((Data.PayloadSpillSize != 0) && "Inconsistent payload stack size"); + + // Peek into the stack. This eventually will become lgc.cps.peek + auto *CspType = getContinuationStackOffsetType(Builder.getContext()); + auto *CspPtr = Builder.CreateCall(getContinuationStackOffset(*Mod)); + auto *Csp = Builder.CreateLoad(CspType, CspPtr); Value *LocalPayloadMem = - moveContinuationStackOffset(B, NeededStackSize).first; + Builder.CreateAdd(Csp, Builder.getInt32(-Data.PayloadSpillSize)); #ifndef NDEBUG // Check that payload pointer exists and is in first position @@ -722,31 +855,21 @@ void LowerRaytracingPipelinePassImpl::replaceContinuationCall( #endif // Copy to payload storage - Value *CastPayload = B.CreateBitCast( + Value *CastPayload = Builder.CreateBitCast( Payload, I32->getPointerTo(Payload->getAddressSpace())); - B.CreateStore(LocalPayloadMem, CastPayload); + Builder.CreateStore(LocalPayloadMem, CastPayload); // Barrier to ensure that accesses to the potentially in-memory parts of // the payload are not re-ordered before this store. More precisely, later // we will insert a load to the payload memory pointer at these accesses. // These loads must be after the store. - B.CreateCall(RegisterBufferSetPointerBarrier, {Payload}); - // Set stacksize metadata on F and functions it will be inlined into - setStacksizeMetadata(*F, OutgoingSerializationInfo->MaxStorageI32s, - MetadataState.getMaxPayloadRegisterCount()); - for (auto *User : F->users()) { - CallInst *CI = dyn_cast(User); - if (!isa(User) || CI->getCalledFunction() != F) - continue; - - setStacksizeMetadata(*CI->getFunction(), - OutgoingSerializationInfo->MaxStorageI32s, - MetadataState.getMaxPayloadRegisterCount()); - } + Builder.CreateCall(RegisterBufferSetPointerBarrier, {Payload}); + // Set stacksize metadata on F + setStacksizeMetadata(*Call->getFunction(), Data.PayloadSpillSize); } // Copy local payload to global payload, before await call (e.g. TraceRay, // CallShader) - copyPayload(B, *PayloadTy, PassedPayload, ShaderStage, PAQAccessKind::Write, - *OutgoingSerializationLayout); + copyPayload(*PayloadOrAttrsTy, PayloadOrAttrs, ShaderStage, + PAQAccessKind::Write, *OutgoingSerializationLayout); } auto *ShaderAddr = Call->getArgOperand(0); @@ -764,10 +887,8 @@ void LowerRaytracingPipelinePassImpl::replaceContinuationCall( assert(TraversalDataTy && "Failed to detect traversal system data type"); SystemDataTy = TraversalDataTy; // Add hit attributes to arguments - auto *HitAttrsArg = F->getArg(F->arg_size() - 1); - auto *HitAttrsTy = PayloadOrAttrTypesForSpecializedFunctions.at(F); - ArgTys.push_back(HitAttrsTy); - auto *HitAttrs = B.CreateLoad(HitAttrsTy, HitAttrsArg); + ArgTys.push_back(PayloadOrAttrsTy); + auto *HitAttrs = Builder.CreateLoad(PayloadOrAttrsTy, PayloadOrAttrs); Args.push_back(HitAttrs); } @@ -775,25 +896,25 @@ void LowerRaytracingPipelinePassImpl::replaceContinuationCall( if (Mutator.shouldRun()) { NewCall = Mutator.insertCpsAwait( Call->getType(), ShaderAddr, Call, Args, CallType, - convertShaderKindToCpsShaderStage(CallerKind)); + convertShaderKindToCpsShaderStage(Data.Kind)); } else { auto *ShaderTy = FunctionType::get(TokenTy, ArgTys, false); - auto *ShaderFun = B.CreateIntToPtr(ShaderAddr, ShaderTy->getPointerTo()); + auto *ShaderFun = + Builder.CreateIntToPtr(ShaderAddr, ShaderTy->getPointerTo()); - auto *Token = B.CreateCall(ShaderTy, ShaderFun, Args); + auto *Token = Builder.CreateCall(ShaderTy, ShaderFun, Args); auto *Await = getContinuationAwait(*Mod, TokenTy, cast(SystemDataTy)); - NewCall = B.CreateCall(Await, {Token}); + NewCall = Builder.CreateCall(Await, {Token}); // Annotate call with the number of registers used for payload DXILContHelper::setOutgoingRegisterCount( - Token, ContinuationStateRegisterCount + - std::min(OutgoingSerializationLayout - ? OutgoingSerializationLayout->NumStorageI32s - : MetadataState.getMaxPayloadRegisterCount(), - MetadataState.getMaxPayloadRegisterCount())); - DXILContHelper::setReturnedRegisterCount( - Token, ContinuationStateRegisterCount + ReturnedRegisterCount.value()); + Token, std::min(OutgoingSerializationLayout + ? OutgoingSerializationLayout->NumStorageI32s + : MetadataState.getMaxPayloadRegisterCount(), + MetadataState.getMaxPayloadRegisterCount())); + DXILContHelper::setReturnedRegisterCount(Token, + ReturnedRegisterCount.value()); // For WaitAwait, add metadata indicating that we wait. After coroutine // passes, we then generate a waitContinue on the awaited function. @@ -801,28 +922,21 @@ void LowerRaytracingPipelinePassImpl::replaceContinuationCall( DXILContHelper::setIsWaitAwaitCall(*Token); } - if (PassedPayload) { + if (CallType != ContinuationCallType::AnyHit) { // Copy global payload back to local payload // Overwrite the local payload with poison first, to make sure it is not // seen as live state. - B.CreateStore(PoisonValue::get(PayloadTy), PassedPayload); + Builder.CreateStore(PoisonValue::get(PayloadOrAttrsTy), PayloadOrAttrs); if (CallType == ContinuationCallType::CallShader) { // For CallShader, there is only a single layout // Copy global payload to local payload, after CallShader call - copyPayload(B, *PayloadTy, PassedPayload, ShaderStage, + copyPayload(*PayloadOrAttrsTy, PayloadOrAttrs, ShaderStage, PAQAccessKind::Read, *OutgoingSerializationLayout); } else { copyTraceRayPayloadIncomingToCaller( - B, *cast(OutgoingSerializationInfo), - PassedPayload); - } - - if (NeededStackSize) { - // Add barrier so no stores that may overwrite the memory pointer are put - // before the payload is read - B.CreateCall(RegisterBufferSetPointerBarrier, {Payload}); - moveContinuationStackOffset(B, -NeededStackSize); + *cast(OutgoingSerializationInfo), + PayloadOrAttrs); } } @@ -837,16 +951,16 @@ void LowerRaytracingPipelinePassImpl::handleReportHit(FunctionData &Data, Function &F) { auto *HitAttrsArg = F.getArg(F.arg_size() - 1); - IRBuilder<> B(F.getContext()); // Look for accept hit calls for (auto &BB : F) { for (auto &I : make_early_inc_range(BB)) { if (auto *Call = dyn_cast(&I)) { - if (Call->getCalledFunction()->getName() == "_AmdAcceptHitAttributes") { + if (Call->getCalledFunction()->getName().starts_with( + "_AmdAcceptHitAttributes")) { // Commit hit attributes - B.SetInsertPoint(Call); + Builder.SetInsertPoint(Call); assert(TraversalDataTy != 0 && "Missing traversal system data!"); - copyHitAttributes(B, Data, Call->getArgOperand(0), TraversalDataTy, + copyHitAttributes(Data, Call->getArgOperand(0), TraversalDataTy, HitAttrsArg, false, nullptr); // Make sure that we store the hit attributes into the correct system // data (just in case dxc copied them around). @@ -861,11 +975,10 @@ void LowerRaytracingPipelinePassImpl::handleReportHit(FunctionData &Data, } /// Replace a call to lgc.rt.shader.index with the passed shader index argument. -void LowerRaytracingPipelinePassImpl::replaceShaderIndexCall(IRBuilder<> &B, - FunctionData &Data, +void LowerRaytracingPipelinePassImpl::replaceShaderIndexCall(FunctionData &Data, CallInst *Call) { if (Data.Kind == DXILShaderKind::RayGeneration) { - Call->replaceAllUsesWith(B.getInt32(0)); + Call->replaceAllUsesWith(Builder.getInt32(0)); } else { auto *ShaderIndex = Call->getFunction()->getArg(ArgShaderIndex); Call->replaceAllUsesWith(ShaderIndex); @@ -880,9 +993,8 @@ void LowerRaytracingPipelinePassImpl::handleGetFuncAddr(Function &Func) { Func.getFunctionType()->getReturnType()->isIntegerTy(32))); auto Name = Func.getName(); - bool Consumed = Name.consume_front("_AmdGetFuncAddr"); + [[maybe_unused]] bool Consumed = Name.consume_front("_AmdGetFuncAddr"); assert(Consumed); - (void)Consumed; Constant *Addr = Mod->getFunction(Name); if (!Addr) @@ -896,6 +1008,42 @@ void LowerRaytracingPipelinePassImpl::handleGetFuncAddr(Function &Func) { }); } +void LowerRaytracingPipelinePassImpl::handleGetShaderKind(Function &Func) { + assert(Func.getReturnType()->isIntegerTy(32) && Func.arg_size() == 0); + + llvm::forEachCall(Func, [&](llvm::CallInst &CInst) { + Function *F = CInst.getFunction(); + auto Stage = lgc::rt::getLgcRtShaderStage(F); + + // Ignore GetShaderKind calls where we cannot find the shader kind. + // This happens e.g. in gpurt-implemented intrinsics that got inlined, + // but not removed. + if (!Stage) + return; + + DXILShaderKind ShaderKind = + DXILContHelper::shaderStageToDxilShaderKind(*Stage); + auto *ShaderKindVal = ConstantInt::get(Func.getReturnType(), + static_cast(ShaderKind)); + CInst.replaceAllUsesWith(ShaderKindVal); + CInst.eraseFromParent(); + }); +} + +void LowerRaytracingPipelinePassImpl::handleGetCurrentFuncAddr(Function &Func) { + assert(Func.arg_size() == 0 && + // Returns an i32 or i64 + (Func.getReturnType()->isIntegerTy(32) || + Func.getReturnType()->isIntegerTy(64))); + + llvm::forEachCall(Func, [&](llvm::CallInst &CInst) { + auto *FuncPtrToInt = + ConstantExpr::getPtrToInt(CInst.getFunction(), Func.getReturnType()); + CInst.replaceAllUsesWith(FuncPtrToInt); + CInst.eraseFromParent(); + }); +} + void llvm::copyBytes(IRBuilder<> &B, Value *Dst, Value *Src, uint64_t NumBytes) { assert(Dst->getType()->isPointerTy() && Src->getType()->isPointerTy() && @@ -938,27 +1086,26 @@ void llvm::copyBytes(IRBuilder<> &B, Value *Dst, Value *Src, } void LowerRaytracingPipelinePassImpl::copyPayload( - IRBuilder<> &B, Type &PayloadTy, Value *LocalPayload, - std::optional Stage, PAQAccessKind GlobalAccessKind, - const PAQSerializationLayout &Layout, + Type &PayloadTy, Value *LocalPayload, std::optional Stage, + PAQAccessKind GlobalAccessKind, const PAQSerializationLayout &Layout, SmallDenseSet *CopiedNodes) { // Nothing to do if there is no serialization type, i.e. the layout is empty if (!Layout.SerializationTy) return; // Obtain pointer to global payload serialization struct - Value *PayloadSerialization = B.CreateBitCast( + Value *PayloadSerialization = Builder.CreateBitCast( Payload, Layout.SerializationTy->getPointerTo(Payload->getAddressSpace())); PayloadCopyHelper Helper{ *Mod, - B, + Builder, PayloadTy, LocalPayload, Stage, GlobalAccessKind, - {B.getInt32(0)}, + {Builder.getInt32(0)}, CopiedNodes, PayloadSerialization, &Layout, @@ -967,7 +1114,7 @@ void LowerRaytracingPipelinePassImpl::copyPayload( } void LowerRaytracingPipelinePassImpl::copyTraceRayPayloadIncomingToCaller( - IRBuilder<> &B, const PAQTraceRaySerializationInfo &SerializationInfo, + const PAQTraceRaySerializationInfo &SerializationInfo, Value *LocalPayload) { SmallDenseSet CopiedNodes; @@ -975,15 +1122,14 @@ void LowerRaytracingPipelinePassImpl::copyTraceRayPayloadIncomingToCaller( PAQSerializationLayoutKind::MissOut}) { const PAQSerializationLayout &Layout = SerializationInfo.LayoutsByKind[LayoutKind]; - copyPayload(B, *SerializationInfo.PayloadRootNode->Ty, LocalPayload, + copyPayload(*SerializationInfo.PayloadRootNode->Ty, LocalPayload, PAQShaderStage::Caller, PAQAccessKind::Read, Layout, &CopiedNodes); } } void LowerRaytracingPipelinePassImpl::savePayloadRegistersBeforeRecursion( - IRBuilder<> &B, DXILShaderKind Kind, - const PAQSerializationLayout &IncomingLayout, + DXILShaderKind Kind, const PAQSerializationLayout &IncomingLayout, const PAQSerializationLayout &OutgoingLayout, SmallVectorImpl &SavedRegisterValues) { @@ -993,7 +1139,7 @@ void LowerRaytracingPipelinePassImpl::savePayloadRegistersBeforeRecursion( SavedRegisterValues.resize(MetadataState.getMaxPayloadRegisterCount()); std::optional Stage = dxilShaderKindToPAQShaderStage(Kind); - auto *RegTy = B.getIntNTy(RegisterBytes * 8); + auto *RegTy = Builder.getIntNTy(RegisterBytes * 8); for (const auto &NodeWithStorageInfo : OutgoingLayout.NodeStorageInfos) { const PAQNode *Node = NodeWithStorageInfo.first; @@ -1018,8 +1164,8 @@ void LowerRaytracingPipelinePassImpl::savePayloadRegistersBeforeRecursion( ++I) { // Create backup of the I-th payload register auto *LoadPtr = - B.CreateConstGEP2_32(Payload->getValueType(), Payload, 0, I); - auto *OldValue = B.CreateLoad(RegTy, LoadPtr); + Builder.CreateConstGEP2_32(Payload->getValueType(), Payload, 0, I); + auto *OldValue = Builder.CreateLoad(RegTy, LoadPtr); // As long as we keep a 32 bit alignment of all fields, all fields // get disjoint registers, and we should never save a register twice. // In case we change that in the future, this assertion will fail, @@ -1039,25 +1185,26 @@ void LowerRaytracingPipelinePassImpl::savePayloadRegistersBeforeRecursion( } void LowerRaytracingPipelinePassImpl::restorePayloadRegistersAfterRecursion( - IRBuilder<> &B, const SmallVectorImpl &SavedRegisterValues) { + const SmallVectorImpl &SavedRegisterValues) { for (unsigned I = 0; I < SavedRegisterValues.size(); ++I) { Value *OldValue = SavedRegisterValues[I]; if (OldValue) { - auto *StorePtr = B.CreateGEP(Payload->getValueType(), Payload, - {B.getInt32(0), B.getInt32(I)}); - B.CreateStore(SavedRegisterValues[I], StorePtr); + auto *StorePtr = + Builder.CreateGEP(Payload->getValueType(), Payload, + {Builder.getInt32(0), Builder.getInt32(I)}); + Builder.CreateStore(SavedRegisterValues[I], StorePtr); } } } void LowerRaytracingPipelinePassImpl::copyHitAttributes( - IRBuilder<> &B, FunctionData &Data, Value *SystemDataPtr, - Type *SystemDataPtrTy, Value *LocalHitAttributes, bool GlobalToLocal, + FunctionData &Data, Value *SystemDataPtr, Type *SystemDataPtrTy, + Value *LocalHitAttributes, bool GlobalToLocal, const PAQSerializationLayout *Layout) { auto *InlineHitAttrsTy = GetTriangleHitAttributes->getReturnType(); - uint64_t InlineHitAttrsBytes = getInlineHitAttrsBytes(*Mod); + uint64_t InlineHitAttrsBytes = getInlineHitAttrsBytes(*GpurtLibrary); uint64_t InlineRegSize = InlineHitAttrsBytes / RegisterBytes; - auto *RegTy = B.getIntNTy(RegisterBytes * 8); + auto *RegTy = Builder.getIntNTy(RegisterBytes * 8); auto *RegTyPtr = RegTy->getPointerTo(); // Hit attribute storage is split between inline hit attributes in system @@ -1067,21 +1214,23 @@ void LowerRaytracingPipelinePassImpl::copyHitAttributes( // to the alloca at the start, or copy back from the alloca to system data, // depending on GlobalToLocal. Then, in the actual copy implementation, we // just access the alloca using loads and stores as for payload registers. - auto InsertPoint = B.saveIP(); - B.SetInsertPoint( - B.GetInsertBlock()->getParent()->getEntryBlock().getFirstNonPHI()); - auto *InlineHitAttrsAlloc = B.CreateAlloca(InlineHitAttrsTy); - B.restoreIP(InsertPoint); - auto *InlineHitAttrs = B.CreateBitCast(InlineHitAttrsAlloc, RegTyPtr); + auto InsertPoint = Builder.saveIP(); + Builder.SetInsertPoint( + Builder.GetInsertBlock()->getParent()->getEntryBlock().getFirstNonPHI()); + auto *InlineHitAttrsAlloc = Builder.CreateAlloca(InlineHitAttrsTy); + Builder.restoreIP(InsertPoint); + auto *InlineHitAttrs = Builder.CreateBitCast(InlineHitAttrsAlloc, RegTyPtr); if (GlobalToLocal) { // Load inline hit attributes from system data auto *SystemDataTy = cast(getFuncArgPtrElementType(GetTriangleHitAttributes, 0)); - auto *SystemData = - getDXILSystemData(B, SystemDataPtr, SystemDataPtrTy, SystemDataTy); - Value *InlineHitAttrs = B.CreateCall(GetTriangleHitAttributes, SystemData); - B.CreateStore(InlineHitAttrs, InlineHitAttrsAlloc); + auto *SystemData = getDXILSystemData(Builder, SystemDataPtr, + SystemDataPtrTy, SystemDataTy); + auto *InlineHitAttrs = + CrossInliner.inlineCall(Builder, GetTriangleHitAttributes, SystemData) + .returnValue; + Builder.CreateStore(InlineHitAttrs, InlineHitAttrsAlloc); } // Hit attribute storage in payload storage @@ -1100,13 +1249,14 @@ void LowerRaytracingPipelinePassImpl::copyHitAttributes( const PAQIndexInterval &IndexInterval = IndexIntervals[0]; // Obtain pointer to global payload serialization struct - Value *PayloadSerialization = B.CreateBitCast( + Value *PayloadSerialization = Builder.CreateBitCast( Payload, Layout->SerializationTy->getPointerTo(Payload->getAddressSpace())); // Last zero yields pointer to the first element of the i32 array - PayloadHitAttrs = B.CreateInBoundsGEP( + PayloadHitAttrs = Builder.CreateInBoundsGEP( Layout->SerializationTy, PayloadSerialization, - {B.getInt32(0), B.getInt32(0), B.getInt32(IndexInterval.Begin)}); + {Builder.getInt32(0), Builder.getInt32(0), + Builder.getInt32(IndexInterval.Begin)}); PayloadHitAttrBytes = RegisterBytes * IndexInterval.size(); } else { // Inline attributes suffice, nothing to do. @@ -1120,8 +1270,8 @@ void LowerRaytracingPipelinePassImpl::copyHitAttributes( Data.FuncConfig.MaxHitAttributeBytes - InlineHitAttrsBytes; // Use hit attribute storage at fixed index PayloadHitAttrs = - B.CreateConstGEP2_32(Payload->getValueType(), Payload, 0, - FirstPayloadHitAttributeStorageRegister); + Builder.CreateConstGEP2_32(Payload->getValueType(), Payload, 0, + FirstPayloadHitAttributeStorageRegister); } uint64_t HitAttrsBytes = @@ -1130,45 +1280,49 @@ void LowerRaytracingPipelinePassImpl::copyHitAttributes( report_fatal_error("Hit attributes are too large!"); assert(InlineHitAttrsBytes + PayloadHitAttrBytes >= HitAttrsBytes && "Insufficient hit attribute storage!"); - LocalHitAttributes = B.CreateBitCast(LocalHitAttributes, RegTyPtr); - auto *I8Ty = B.getInt8Ty(); + LocalHitAttributes = Builder.CreateBitCast(LocalHitAttributes, RegTyPtr); + auto *I8Ty = Builder.getInt8Ty(); for (unsigned I = 0; I < divideCeil(HitAttrsBytes, RegisterBytes); I++) { - auto *LocalPtr = B.CreateConstInBoundsGEP1_64(RegTy, LocalHitAttributes, I); + auto *LocalPtr = + Builder.CreateConstInBoundsGEP1_64(RegTy, LocalHitAttributes, I); Value *GlobalPtr; if (I < InlineRegSize) - GlobalPtr = B.CreateConstInBoundsGEP1_64(RegTy, InlineHitAttrs, I); + GlobalPtr = Builder.CreateConstInBoundsGEP1_64(RegTy, InlineHitAttrs, I); else - GlobalPtr = B.CreateConstInBoundsGEP1_64(RegTy, PayloadHitAttrs, - I - InlineRegSize); + GlobalPtr = Builder.CreateConstInBoundsGEP1_64(RegTy, PayloadHitAttrs, + I - InlineRegSize); auto *LoadPtr = GlobalToLocal ? GlobalPtr : LocalPtr; auto *StorePtr = GlobalToLocal ? LocalPtr : GlobalPtr; if ((I + 1) * RegisterBytes <= HitAttrsBytes) { // Can load a whole register - auto *Val = B.CreateLoad(RegTy, LoadPtr); - B.CreateStore(Val, StorePtr); + auto *Val = Builder.CreateLoad(RegTy, LoadPtr); + Builder.CreateStore(Val, StorePtr); } else { // Load byte by byte into a vector and pad the rest with undef - auto *ByteLoadPtr = B.CreateBitCast(LoadPtr, I8Ty->getPointerTo()); - auto *ByteStorePtr = B.CreateBitCast(StorePtr, I8Ty->getPointerTo()); + auto *ByteLoadPtr = Builder.CreateBitCast(LoadPtr, I8Ty->getPointerTo()); + auto *ByteStorePtr = + Builder.CreateBitCast(StorePtr, I8Ty->getPointerTo()); for (unsigned J = 0; J < HitAttrsBytes % RegisterBytes; J++) { - auto *Val = B.CreateLoad( - I8Ty, B.CreateConstInBoundsGEP1_64(I8Ty, ByteLoadPtr, J)); - B.CreateStore(Val, B.CreateConstInBoundsGEP1_64(I8Ty, ByteStorePtr, J)); + auto *Val = Builder.CreateLoad( + I8Ty, Builder.CreateConstInBoundsGEP1_64(I8Ty, ByteLoadPtr, J)); + Builder.CreateStore( + Val, Builder.CreateConstInBoundsGEP1_64(I8Ty, ByteStorePtr, J)); } } } if (!GlobalToLocal) { // Store inline hit attributes to system data - auto *Attrs = B.CreateLoad(InlineHitAttrsTy, InlineHitAttrsAlloc); + auto *Attrs = Builder.CreateLoad(InlineHitAttrsTy, InlineHitAttrsAlloc); auto *SystemDataTy = cast(getFuncArgPtrElementType(GetTriangleHitAttributes, 0)); - auto *SystemData = - getDXILSystemData(B, SystemDataPtr, SystemDataPtrTy, SystemDataTy); + auto *SystemData = getDXILSystemData(Builder, SystemDataPtr, + SystemDataPtrTy, SystemDataTy); assert(SetTriangleHitAttributes && "Could not find SetTriangleHitAttributes function"); - B.CreateCall(SetTriangleHitAttributes, {SystemData, Attrs}); + CrossInliner.inlineCall(Builder, SetTriangleHitAttributes, + {SystemData, Attrs}); } } @@ -1213,10 +1367,9 @@ void LowerRaytracingPipelinePassImpl::createPayloadGlobal() { } void LowerRaytracingPipelinePassImpl::setTraversalRegisterCountMetadata() { - const uint32_t NumPayloadI32s = std::min( + const uint32_t NumI32s = std::min( static_cast(Payload->getValueType()->getArrayNumElements()), MetadataState.getMaxPayloadRegisterCount()); - const uint32_t NumI32s = NumPayloadI32s + ContinuationStateRegisterCount; // Find traversal functions without walking over all functions by checking // uses of the `continuation.[wait]continue` intrinsics. @@ -1231,7 +1384,8 @@ void LowerRaytracingPipelinePassImpl::setTraversalRegisterCountMetadata() { continue; auto *TraversalVariant = CI->getFunction(); - if (!DXILContHelper::isTraversal(*TraversalVariant)) + auto Stage = lgc::rt::getLgcRtShaderStage(TraversalVariant); + if (!Stage || *Stage != lgc::rt::RayTracingShaderStage::Traversal) continue; assert(!DXILContHelper::tryGetOutgoingRegisterCount(CI).has_value() && @@ -1250,35 +1404,41 @@ void LowerRaytracingPipelinePassImpl::processContinuations() { TokenTy = StructType::create(*Context, "continuation.token")->getPointerTo(); RegisterBufferSetPointerBarrier = getRegisterBufferSetPointerBarrier(*Mod); - llvm_dialects::Builder B(*Context); for (auto &FuncData : ToProcess) { - processFunction(B, FuncData.first, FuncData.second); + processFunction(FuncData.first, FuncData.second); } } -void LowerRaytracingPipelinePassImpl::processFunctionEntry( - llvm_dialects::Builder &B, Function *F, FunctionData &Data) { +void LowerRaytracingPipelinePassImpl::processFunctionEntry(Function *F, + FunctionData &Data) { // Create system data // See also the system data documentation at the top of Continuations.h. - Data.SystemData = B.CreateAlloca(Data.SystemDataTy); + Data.SystemData = Builder.CreateAlloca(Data.SystemDataTy); Data.SystemData->setName("system.data.alloca"); // Initialize system data by calling the getSystemData intrinsic auto *SystemDataIntr = - B.create(Data.SystemDataTy); - B.CreateStore(SystemDataIntr, Data.SystemData); + Builder.create(Data.SystemDataTy); + Builder.CreateStore(SystemDataIntr, Data.SystemData); // Set local root signature on entry assert(GetLocalRootIndex && "Could not find GetLocalRootIndex function"); - auto *LocalIndex = B.CreateCall( - GetLocalRootIndex, - getDXILSystemData(B, Data.SystemData, Data.SystemDataTy, - getFuncArgPtrElementType(GetLocalRootIndex, 0))); + auto *LocalIndex = + CrossInliner + .inlineCall( + Builder, GetLocalRootIndex, + getDXILSystemData(Builder, Data.SystemData, Data.SystemDataTy, + getFuncArgPtrElementType(GetLocalRootIndex, 0))) + .returnValue; LocalIndex->setName("local.root.index"); - B.CreateCall(SetLocalRootIndex, LocalIndex); + Builder.CreateCall(SetLocalRootIndex, LocalIndex); + + // Allocate payload spilling space + if (Data.PayloadSpillSize > 0) + moveContinuationStackOffset(Builder, Data.PayloadSpillSize); } void LowerRaytracingPipelinePassImpl::processFunctionEnd( - llvm_dialects::Builder &B, FunctionData &Data, FunctionEndData &EData) { + FunctionData &Data, FunctionEndData &EData) { AnyHitExitKind AHExitKind = AnyHitExitKind::None; bool IsAnyHit = Data.Kind == DXILShaderKind::AnyHit; @@ -1301,7 +1461,7 @@ void LowerRaytracingPipelinePassImpl::processFunctionEnd( } } - B.SetInsertPoint(EData.Terminator); + Builder.SetInsertPoint(EData.Terminator); auto *PayloadTy = Data.IncomingPayload; if (Data.Kind != DXILShaderKind::RayGeneration && @@ -1314,9 +1474,9 @@ void LowerRaytracingPipelinePassImpl::processFunctionEnd( assert(AcceptHit && "Could not find AcceptHit function"); auto *SystemDataTy = cast(getFuncArgPtrElementType(AcceptHit, 0)); - auto *SystemData = getDXILSystemData(B, Data.SystemData, + auto *SystemData = getDXILSystemData(Builder, Data.SystemData, Data.SystemDataTy, SystemDataTy); - B.CreateCall(AcceptHit, SystemData); + CrossInliner.inlineCall(Builder, AcceptHit, SystemData); } EData.OutgoingSerializationLayout = @@ -1328,12 +1488,12 @@ void LowerRaytracingPipelinePassImpl::processFunctionEnd( // Restore saved registers. This needs to be done *before* copying // back the payload, which depends on the restored memory pointer! - restorePayloadRegistersAfterRecursion(B, EData.SavedRegisterValues); + restorePayloadRegistersAfterRecursion(EData.SavedRegisterValues); // Copy local payload into global payload at end of shader if (EData.OutgoingSerializationLayout->NumStorageI32s) { - B.CreateCall(RegisterBufferSetPointerBarrier, {Payload}); - copyPayload(B, *PayloadTy, EData.NewPayload, EData.ShaderStage, + Builder.CreateCall(RegisterBufferSetPointerBarrier, {Payload}); + copyPayload(*PayloadTy, EData.NewPayload, EData.ShaderStage, PAQAccessKind::Write, *EData.OutgoingSerializationLayout); } @@ -1344,24 +1504,28 @@ void LowerRaytracingPipelinePassImpl::processFunctionEnd( // TODO Only if there is a ClosestHit shader in any hit group // where this AnyHit is used. If there is no ClosestHit, the // attributes can never be read, so we don't need to store them. - copyHitAttributes(B, Data, Data.SystemData, Data.SystemDataTy, + copyHitAttributes(Data, Data.SystemData, Data.SystemDataTy, EData.HitAttrsAlloca, false, EData.OutgoingSerializationLayout); } else { assert(AHExitKind == AnyHitExitKind::IgnoreHit); // Copy original hit attributes - copyHitAttributes(B, Data, Data.SystemData, Data.SystemDataTy, + copyHitAttributes(Data, Data.SystemData, Data.SystemDataTy, EData.OrigHitAttrsAlloca, false, EData.OutgoingSerializationLayout); } } } + if (Data.PayloadSpillSize > 0) + moveContinuationStackOffset(Builder, -Data.PayloadSpillSize); + Value *RetValue = nullptr; if (!Data.ReturnTy->isVoidTy()) { - auto *SystemData = getDXILSystemData(B, Data.SystemData, Data.SystemDataTy, - cast(Data.ReturnTy)); - RetValue = B.CreateLoad(Data.ReturnTy, SystemData); + auto *SystemData = + getDXILSystemData(Builder, Data.SystemData, Data.SystemDataTy, + cast(Data.ReturnTy)); + RetValue = Builder.CreateLoad(Data.ReturnTy, SystemData); } if (Mutator.shouldRun()) { @@ -1374,35 +1538,34 @@ void LowerRaytracingPipelinePassImpl::processFunctionEnd( if (Data.Kind == DXILShaderKind::RayGeneration) { assert(RetArgs.empty() && "RayGen cannot return anything"); - B.CreateRetVoid(); + Builder.CreateRetVoid(); } else { - B.create( + Builder.create( EData.Terminator->getFunction()->getArg(ArgReturnAddr), CpsRetLevel, - PoisonValue::get(StructType::get(B.getContext())), RetArgs); - B.CreateUnreachable(); + PoisonValue::get(StructType::get(Builder.getContext())), RetArgs); + Builder.CreateUnreachable(); } } else { - Instruction *Ret = RetValue ? B.CreateRet(RetValue) : B.CreateRetVoid(); + Instruction *Ret = + RetValue ? Builder.CreateRet(RetValue) : Builder.CreateRetVoid(); // Annotate ret with number of outgoing payload registers. // This annotation will be passed along the following transformations, // ending up at the final continuation call. unsigned OutgoingRegisterCount = - ContinuationStateRegisterCount + - (EData.OutgoingSerializationLayout - ? std::min(EData.OutgoingSerializationLayout->NumStorageI32s, - MetadataState.getMaxPayloadRegisterCount()) - : MetadataState.getMaxPayloadRegisterCount()); + EData.OutgoingSerializationLayout + ? std::min(EData.OutgoingSerializationLayout->NumStorageI32s, + MetadataState.getMaxPayloadRegisterCount()) + : MetadataState.getMaxPayloadRegisterCount(); DXILContHelper::setOutgoingRegisterCount(Ret, OutgoingRegisterCount); } EData.Terminator->eraseFromParent(); } -void LowerRaytracingPipelinePassImpl::processFunction(llvm_dialects::Builder &B, - Function *F, +void LowerRaytracingPipelinePassImpl::processFunction(Function *F, FunctionData &Data) { - B.SetInsertPointPastAllocas(F); + Builder.SetInsertPointPastAllocas(F); // Change the return type and arguments for shaders that are not RayGen SmallVector AllArgTypes; @@ -1419,14 +1582,14 @@ void LowerRaytracingPipelinePassImpl::processFunction(llvm_dialects::Builder &B, // * Remaining arguments (system data, optionally hit attributes) AllArgTypes.push_back(StructType::get(Mod->getContext())); - AllArgTypes.push_back(B.getInt32Ty()); - AllArgTypes.push_back(B.getInt32Ty()); + AllArgTypes.push_back(Builder.getInt32Ty()); + AllArgTypes.push_back(Builder.getInt32Ty()); } if (Data.Kind == DXILShaderKind::RayGeneration) { assert(SetupRayGen && "Could not find SetupRayGen function"); SystemDataTy = SetupRayGen->getReturnType(); - NewRetTy = B.getVoidTy(); + NewRetTy = Builder.getVoidTy(); } else { switch (Data.Kind) { case DXILShaderKind::Intersection: { @@ -1465,13 +1628,19 @@ void LowerRaytracingPipelinePassImpl::processFunction(llvm_dialects::Builder &B, } } + Data.PayloadSpillSize = computeNeededStackSizeForRegisterBuffer( + Data.MaxOutgoingPayloadI32s, MetadataState.getMaxPayloadRegisterCount()); + assert((Data.PayloadSpillSize == 0) || + (Data.Kind != DXILShaderKind::Intersection)); Data.SystemDataTy = cast(SystemDataTy); - processFunctionEntry(B, F, Data); + processFunctionEntry(F, Data); - auto *FunctionTypeRetTy = Mutator.shouldRun() ? B.getVoidTy() : NewRetTy; + auto *FunctionTypeRetTy = + Mutator.shouldRun() ? Builder.getVoidTy() : NewRetTy; // Create new function to change signature auto *NewFuncTy = FunctionType::get(FunctionTypeRetTy, AllArgTypes, false); - Function *NewFunc = cloneFunctionHeader(*F, NewFuncTy, {}); + Function *NewFunc = CompilerUtils::cloneFunctionHeader( + *F, NewFuncTy, ArrayRef{}); NewFunc->takeName(F); llvm::moveFunctionBody(*F, *NewFunc); @@ -1538,9 +1707,9 @@ void LowerRaytracingPipelinePassImpl::processFunction(llvm_dialects::Builder &B, { // Preserve current insert point - IRBuilder<>::InsertPointGuard Guard(B); - B.SetInsertPointPastAllocas(NewFunc); - NewPayload = B.CreateAlloca(PayloadTy); + IRBuilder<>::InsertPointGuard Guard(Builder); + Builder.SetInsertPointPastAllocas(NewFunc); + NewPayload = Builder.CreateAlloca(PayloadTy); FPayload->replaceAllUsesWith(NewPayload); } @@ -1549,24 +1718,23 @@ void LowerRaytracingPipelinePassImpl::processFunction(llvm_dialects::Builder &B, } else { // Annotate function with the number of registers for incoming payload DXILContHelper::setIncomingRegisterCount( - NewFunc, ContinuationStateRegisterCount + - std::min(IncomingSerializationLayout.NumStorageI32s, - MetadataState.getMaxPayloadRegisterCount())); + NewFunc, std::min(IncomingSerializationLayout.NumStorageI32s, + MetadataState.getMaxPayloadRegisterCount())); // Copy global payload into local payload at start of shader if (IncomingSerializationLayout.NumStorageI32s) { - copyPayload(B, *PayloadTy, NewPayload, ShaderStage, - PAQAccessKind::Read, IncomingSerializationLayout); + copyPayload(*PayloadTy, NewPayload, ShaderStage, PAQAccessKind::Read, + IncomingSerializationLayout); // Add barrier so no stores that may overwrite the memory pointer // are put before the payload is read - B.CreateCall(RegisterBufferSetPointerBarrier, {Payload}); + Builder.CreateCall(RegisterBufferSetPointerBarrier, {Payload}); } if (!Data.CallShaderCalls.empty() || !Data.TraceRayCalls.empty()) { assert(OutgoingSerializationLayout && "Missing outgoing serialization layout!"); savePayloadRegistersBeforeRecursion( - B, Data.Kind, IncomingSerializationLayout, + Data.Kind, IncomingSerializationLayout, *OutgoingSerializationLayout, SavedRegisterValues); } } @@ -1578,23 +1746,24 @@ void LowerRaytracingPipelinePassImpl::processFunction(llvm_dialects::Builder &B, { // Preserve current insert point - IRBuilder<>::InsertPointGuard Guard(B); - B.SetInsertPointPastAllocas(NewFunc); - OrigHitAttrsAlloca = B.CreateAlloca(ArrayType::get( + IRBuilder<>::InsertPointGuard Guard(Builder); + Builder.SetInsertPointPastAllocas(NewFunc); + OrigHitAttrsAlloca = Builder.CreateAlloca(ArrayType::get( I32, divideCeil(GlobalMaxHitAttributeBytes, RegisterBytes))); OrigHitAttrsAlloca->setName("OrigHitAttrs"); - HitAttrsAlloca = B.CreateAlloca(Data.HitAttributes); + HitAttrsAlloca = Builder.CreateAlloca(Data.HitAttributes); HitAttrsAlloca->setName("HitAttrsAlloca"); } // Copy old hit attributes from payload - copyHitAttributes(B, Data, Data.SystemData, Data.SystemDataTy, + copyHitAttributes(Data, Data.SystemData, Data.SystemDataTy, OrigHitAttrsAlloca, true, &IncomingSerializationLayout); // Copy new hit attributes from argument - B.CreateStore(NewFunc->getArg(NewFunc->arg_size() - 1), HitAttrsAlloca); + Builder.CreateStore(NewFunc->getArg(NewFunc->arg_size() - 1), + HitAttrsAlloca); HitAttrs->replaceAllUsesWith(HitAttrsAlloca); } else if (Data.Kind == DXILShaderKind::ClosestHit) { assert(F->arg_size() == 2 && "Shader has more arguments than expected"); @@ -1603,17 +1772,17 @@ void LowerRaytracingPipelinePassImpl::processFunction(llvm_dialects::Builder &B, Value *NewHitAttrs; { // Preserve current insert point - IRBuilder<>::InsertPointGuard Guard(B); - B.SetInsertPointPastAllocas(NewFunc); - NewHitAttrs = B.CreateAlloca(Data.HitAttributes); + IRBuilder<>::InsertPointGuard Guard(Builder); + Builder.SetInsertPointPastAllocas(NewFunc); + NewHitAttrs = Builder.CreateAlloca(Data.HitAttributes); NewHitAttrs->setName("HitAttrs"); } // Copy hit attributes from system data and payload into the local // variable OrigHitAttrs->replaceAllUsesWith(NewHitAttrs); - copyHitAttributes(B, Data, Data.SystemData, Data.SystemDataTy, - NewHitAttrs, true, &IncomingSerializationLayout); + copyHitAttributes(Data, Data.SystemData, Data.SystemDataTy, NewHitAttrs, + true, &IncomingSerializationLayout); } } else { if (!Mutator.shouldRun()) { @@ -1624,8 +1793,7 @@ void LowerRaytracingPipelinePassImpl::processFunction(llvm_dialects::Builder &B, // that instead. For a library compile, we can't know the max // payload size of shaders in pipelines this shader is used in. DXILContHelper::setIncomingRegisterCount( - NewFunc, ContinuationStateRegisterCount + - MetadataState.getMaxPayloadRegisterCount()); + NewFunc, MetadataState.getMaxPayloadRegisterCount()); } } @@ -1639,14 +1807,18 @@ void LowerRaytracingPipelinePassImpl::processFunction(llvm_dialects::Builder &B, Data.ReturnTy = NewRetTy; // Modify function ends - for (auto &BB : *NewFunc) { - auto *I = BB.getTerminator(); + // While iterating over function ends, basic blocks are inserted by inlining + // functions, so we copy them beforehand. + SmallVector BBs(make_pointer_range(*NewFunc)); + for (auto *BB : BBs) { + auto *I = BB->getTerminator(); + assert(I && "BB must have terminator"); // Replace the end of the BB if it terminates the function bool IsFunctionEnd = (I->getOpcode() == Instruction::Ret || I->getOpcode() == Instruction::Unreachable); if (IsFunctionEnd) { EData.Terminator = I; - processFunctionEnd(B, Data, EData); + processFunctionEnd(Data, EData); } } @@ -1660,32 +1832,34 @@ void LowerRaytracingPipelinePassImpl::processFunction(llvm_dialects::Builder &B, // Replace TraceRay calls for (auto *Call : Data.TraceRayCalls) { - B.SetInsertPoint(&*++Call->getIterator()); - replaceTraceRayCall(B, Data, Call); + assert(TraceRay && "TraceRay not found"); + Builder.SetInsertPoint(&*++Call->getIterator()); + replaceCall(Data, Call, TraceRay, ContinuationCallType::Traversal); } // Replace ReportHit calls for (auto *Call : Data.ReportHitCalls) { - B.SetInsertPoint(&*++Call->getIterator()); - replaceReportHitCall(B, Data, Call); + Builder.SetInsertPoint(&*++Call->getIterator()); + replaceReportHitCall(Data, Call); } // Replace CallShader calls for (auto *Call : Data.CallShaderCalls) { - B.SetInsertPoint(&*++Call->getIterator()); - replaceCallShaderCall(B, Data, Call); + assert(CallShader && "CallShader not found"); + Builder.SetInsertPoint(&*++Call->getIterator()); + replaceCall(Data, Call, CallShader, ContinuationCallType::CallShader); } // Replace ShaderIndexOp calls for (auto *Call : Data.ShaderIndexCalls) { - B.SetInsertPoint(&*++Call->getIterator()); - replaceShaderIndexCall(B, Data, Call); + Builder.SetInsertPoint(&*++Call->getIterator()); + replaceShaderIndexCall(Data, Call); } // Replace non-rematerializable intrinsic calls for (auto *Call : Data.IntrinsicCalls) - replaceIntrinsicCall(B, Data.SystemDataTy, Data.SystemData, Data.Kind, - Call); + replaceIntrinsicCall(Builder, Data.SystemDataTy, Data.SystemData, Data.Kind, + Call, GpurtLibrary, CrossInliner); #ifndef NDEBUG if (!Mutator.shouldRun() && Data.Kind != DXILShaderKind::RayGeneration) { @@ -1755,7 +1929,6 @@ void LowerRaytracingPipelinePassImpl::collectProcessableFunctions() { if (Kind == DXILShaderKind::Intersection) { Data.MaxOutgoingPayloadI32s = - ContinuationStateRegisterCount + MetadataState.getMaxPayloadRegisterCount(); } @@ -1841,24 +2014,40 @@ void LowerRaytracingPipelinePassImpl::handleDriverFuncAssertions() { void LowerRaytracingPipelinePassImpl::handleAmdInternalFunc(Function &Func) { StringRef FuncName = Func.getName(); - if (FuncName.starts_with("_AmdAwait") || - FuncName.starts_with("_AmdWaitAwait")) { - Awaits.push_back(&Func); - assert(!Func.arg_empty() - // Function address - && Func.getFunctionType()->getParamType(0) == - Type::getInt64Ty(*Context)); - } - if (FuncName.starts_with("_AmdRestoreSystemData")) { - RestoreSystemDatas.push_back(&Func); assert(Func.arg_size() == 1 // Function address && Func.getFunctionType()->getParamType(0)->isPointerTy()); + llvm::forEachCall(Func, [&](llvm::CallInst &CInst) { + Builder.SetInsertPoint(&CInst); + handleRestoreSystemData(&CInst); + }); + } else if (FuncName.starts_with("_AmdGetFuncAddr")) { + handleGetFuncAddr(Func); + } else if (FuncName.starts_with("_AmdGetShaderKind")) { + handleGetShaderKind(Func); + } else if (FuncName.starts_with("_AmdGetCurrentFuncAddr")) { + handleGetCurrentFuncAddr(Func); } +} - if (FuncName.starts_with("_AmdGetFuncAddr")) { - handleGetFuncAddr(Func); +// Split BB after _AmdRestoreSystemData. +// The coroutine passes rematerialize to the start of the basic block of a use. +// We split the block so that every rematerialized dxil intrinsic lands after +// the restore call and accesses the restored system data. +// If we did not do that, an intrinsic that is rematerialized to before +// RestoreSystemData is called gets an uninitialized system data struct as +// argument. +void LowerRaytracingPipelinePassImpl::splitRestoreBB() { + for (auto &F : *Mod) { + if (F.getName().startswith("_AmdRestoreSystemData")) { + llvm::forEachCall(F, [](llvm::CallInst &CInst) { + auto *Next = &*++CInst.getIterator(); + CInst.eraseFromParent(); + if (!Next->isTerminator()) + SplitBlock(Next->getParent(), Next); + }); + } } } @@ -1875,7 +2064,8 @@ void LowerRaytracingPipelinePassImpl::handleUnrematerializableCandidates() { llvm::forEachCall(Func, [&](llvm::CallInst &CInst) { auto Data = ToProcess.find(CInst.getFunction()); if (Data != ToProcess.end()) { - if (!isRematerializableLgcRtOp(CInst, Data->second.Kind)) + if (!DXILContHelper::isRematerializableLgcRtOp(CInst, + Data->second.Kind)) Data->second.IntrinsicCalls.push_back(&CInst); } }); @@ -1884,25 +2074,26 @@ void LowerRaytracingPipelinePassImpl::handleUnrematerializableCandidates() { } void LowerRaytracingPipelinePassImpl::collectDriverFunctions() { - IsEndSearch = Mod->getFunction("_cont_IsEndSearch"); - GetTriangleHitAttributes = Mod->getFunction("_cont_GetTriangleHitAttributes"); - SetTriangleHitAttributes = Mod->getFunction("_cont_SetTriangleHitAttributes"); - GetLocalRootIndex = Mod->getFunction("_cont_GetLocalRootIndex"); + IsEndSearch = GpurtLibrary->getFunction("_cont_IsEndSearch"); + GetTriangleHitAttributes = + GpurtLibrary->getFunction("_cont_GetTriangleHitAttributes"); + SetTriangleHitAttributes = + GpurtLibrary->getFunction("_cont_SetTriangleHitAttributes"); + GetLocalRootIndex = GpurtLibrary->getFunction("_cont_GetLocalRootIndex"); SetLocalRootIndex = getSetLocalRootIndex(*Mod); - SetupRayGen = Mod->getFunction("_cont_SetupRayGen"); - TraceRay = Mod->getFunction("_cont_TraceRay"); - // TODO Temporarily support multiple prefixes for this function - if (!TraceRay) - TraceRay = Mod->getFunction("amd.dx.TraceRay"); - CallShader = Mod->getFunction("_cont_CallShader"); - ReportHit = Mod->getFunction("_cont_ReportHit"); - AcceptHit = Mod->getFunction("_cont_AcceptHit"); + SetupRayGen = GpurtLibrary->getFunction("_cont_SetupRayGen"); + TraceRay = GpurtLibrary->getFunction("_cont_TraceRay"); + CallShader = GpurtLibrary->getFunction("_cont_CallShader"); + ReportHit = GpurtLibrary->getFunction("_cont_ReportHit"); + AcceptHit = GpurtLibrary->getFunction("_cont_AcceptHit"); } + LowerRaytracingPipelinePassImpl::LowerRaytracingPipelinePassImpl( - llvm::Module &M) - : Mod{&M}, Context{&M.getContext()}, DL{&M.getDataLayout()}, - MetadataState{*Mod}, Mutator{*Mod}, - PAQManager{Mod, MetadataState.getMaxPayloadRegisterCount()} {} + llvm::Module &M, Module &GpurtLibrary) + : Mod{&M}, GpurtLibrary{&GpurtLibrary}, Context{&M.getContext()}, + DL{&M.getDataLayout()}, Builder{Mod->getContext()}, MetadataState{*Mod}, + Mutator{*Mod}, PAQManager{Mod, &GpurtLibrary, + MetadataState.getMaxPayloadRegisterCount()} {} bool LowerRaytracingPipelinePassImpl::run() { MetadataState.updateModuleMetadata(); @@ -1971,12 +2162,6 @@ bool LowerRaytracingPipelinePassImpl::run() { VisitorState S{PAQManager, ToProcess}; Visitor.visit(S, *Mod); - for (auto &Func : *Mod) { - if (Func.getName().starts_with("_Amd")) { - handleAmdInternalFunc(Func); - } - } - handleUnrematerializableCandidates(); handleDriverFuncAssertions(); @@ -1986,7 +2171,7 @@ bool LowerRaytracingPipelinePassImpl::run() { if (ReportHit) TraversalDataTy = getFuncArgPtrElementType(ReportHit, 0); HitMissDataTy = nullptr; - if (auto *HitKind = Mod->getFunction("_cont_HitKind")) { + if (auto *HitKind = GpurtLibrary->getFunction("_cont_HitKind")) { HitMissDataTy = getFuncArgPtrElementType(HitKind, 0); LLVM_DEBUG(dbgs() << "HitMiss system data from _cont_HitKind: "; HitMissDataTy->dump()); @@ -1997,31 +2182,23 @@ bool LowerRaytracingPipelinePassImpl::run() { processContinuations(); - // Handle places after Awaits where system data is restored - IRBuilder<> B(*Context); - for (llvm::Function *Func : RestoreSystemDatas) { - llvm::forEachCall(*Func, [&](llvm::CallInst &CInst) { - B.SetInsertPoint(&CInst); - handleRestoreSystemData(B, &CInst); - }); + for (auto &Func : *Mod) { + if (Func.getName().starts_with("_Amd")) { + handleAmdInternalFunc(Func); + } } - // Change specialized functions afterwards, so the payload or hit attributes - // exist as the last argument - for (llvm::Function *Func : Awaits) { - llvm::forEachCall(*Func, [&](llvm::CallInst &CInst) { - auto Data = AwaitsToProcess.find(CInst.getFunction()); - if (Data != AwaitsToProcess.end()) - Data->second.AwaitCalls.push_back(&CInst); - }); - } + splitRestoreBB(); - for (auto &FuncData : AwaitsToProcess) { - for (auto *Call : FuncData.second.AwaitCalls) { - B.SetInsertPoint(Call); - replaceContinuationCall(B, FuncData.second.CallType, Call, - FuncData.second.FuncConfig, - FuncData.second.CallerKind); + if (Mod == GpurtLibrary) { + // For tests, remove intrinsic implementations from the module + for (auto &F : make_early_inc_range(*Mod)) { + auto Name = F.getName(); + if (Name.startswith("_cont_TraceRay") || + Name.startswith("_cont_CallShader") || + Name.startswith("_cont_ReportHit")) { + F.eraseFromParent(); + } } } @@ -2057,4 +2234,16 @@ llvm::dxilShaderKindToPAQShaderStage(DXILShaderKind ShaderKind) { default: return {}; } +} // anonymous namespace + +llvm::PreservedAnalyses +LowerRaytracingPipelinePass::run(llvm::Module &M, + llvm::ModuleAnalysisManager &AnalysisManager) { + LLVM_DEBUG(dbgs() << "Run the pass lower-raytracing-pipeline\n"); + AnalysisManager.getResult(M); + + LowerRaytracingPipelinePassImpl Impl(M, GpurtLibrary ? *GpurtLibrary : M); + bool Changed = Impl.run(); + + return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); } diff --git a/shared/continuations/lib/PassRegistry.inc b/shared/continuations/lib/PassRegistry.inc index 74e8d4d9ec..ce65a5559e 100644 --- a/shared/continuations/lib/PassRegistry.inc +++ b/shared/continuations/lib/PassRegistry.inc @@ -47,14 +47,15 @@ CONT_MODULE_ANALYSIS("dialect-context-analysis", DialectContextAnalysis(NeedDialectContext)) -CONT_MODULE_PASS("legacy-cleanup-continuations", LegacyCleanupContinuationsPass()) +CONT_MODULE_PASS("legacy-cleanup-continuations", + LegacyCleanupContinuationsPass()) CONT_MODULE_PASS("cleanup-continuations", CleanupContinuationsPass()) CONT_MODULE_PASS("lower-raytracing-pipeline", LowerRaytracingPipelinePass()) CONT_MODULE_PASS("dxil-cont-intrinsic-prepare", DXILContIntrinsicPreparePass()) -CONT_MODULE_PASS("dxil-cont-lgc-rt-op-converter", DXILContLgcRtOpConverterPass()) +CONT_MODULE_PASS("dxil-cont-lgc-rt-op-converter", + DXILContLgcRtOpConverterPass()) CONT_MODULE_PASS("dxil-cont-post-process", DXILContPostProcessPass()) CONT_MODULE_PASS("dxil-cont-post-hook", DXILContPostHookPass()) -CONT_MODULE_PASS("pre-coroutine-lowering", PreCoroutineLoweringPass()) CONT_MODULE_PASS("dxil-cont-pre-hook", DXILContPreHookPass()) CONT_MODULE_PASS("lower-await", LowerAwaitPass()) CONT_MODULE_PASS("register-buffer", RegisterBufferPass()) diff --git a/shared/continuations/lib/PayloadAccessQualifiers.cpp b/shared/continuations/lib/PayloadAccessQualifiers.cpp index affdde5003..35fa424736 100644 --- a/shared/continuations/lib/PayloadAccessQualifiers.cpp +++ b/shared/continuations/lib/PayloadAccessQualifiers.cpp @@ -1852,8 +1852,9 @@ PAQCallShaderSerializationInfo::create(Module &M, } PAQSerializationInfoManager::PAQSerializationInfoManager( - Module *M, uint32_t MaxPayloadRegCount) - : Mod{M}, MaxPayloadRegisterCount(MaxPayloadRegCount) { + Module *M, Module *GpurtLibrary, uint32_t MaxPayloadRegCount) + : Mod{M}, GpurtLibrary{GpurtLibrary}, + MaxPayloadRegisterCount(MaxPayloadRegCount) { TraceRayCache.PAQRootNodes = importModulePayloadPAQNodes(*M); } @@ -1879,7 +1880,7 @@ PAQTraceRaySerializationInfo & PAQSerializationInfoManager::getOrCreateTraceRaySerializationInfo( const PAQPayloadConfig &PAQConfig) { return TraceRayCache.getOrCreateSerializationInfo( - *Mod, MaxPayloadRegisterCount, PAQConfig); + *GpurtLibrary, MaxPayloadRegisterCount, PAQConfig); } PAQCallShaderSerializationInfo & @@ -1889,7 +1890,7 @@ PAQSerializationInfoManager::getOrCreateCallShaderSerializationInfo( PAQPayloadConfig PAQConfigWithRelevantData = PAQConfig; PAQConfigWithRelevantData.MaxHitAttributeByteCount = 0; return CallShaderCache.getOrCreateSerializationInfo( - *Mod, MaxPayloadRegisterCount, PAQConfigWithRelevantData); + *GpurtLibrary, MaxPayloadRegisterCount, PAQConfigWithRelevantData); } template @@ -1965,7 +1966,7 @@ PAQSerializationInfoManager::getOrCreateTraceRayLayout( Mod->getDataLayout().getTypeStoreSize(HitAttributesTy).getFixedValue(); if (AttrsBytes > TraceRayInfo.PAQConfig.MaxHitAttributeByteCount) report_fatal_error("Hit attributes are too large!"); - uint64_t InlineHitAttrsBytes = getInlineHitAttrsBytes(*Mod); + uint64_t InlineHitAttrsBytes = getInlineHitAttrsBytes(*GpurtLibrary); uint64_t AttrsInPayloadBytes = AttrsBytes > InlineHitAttrsBytes ? AttrsBytes - InlineHitAttrsBytes : 0; diff --git a/shared/continuations/lib/PreCoroutineLowering.cpp b/shared/continuations/lib/PreCoroutineLowering.cpp deleted file mode 100644 index 7ec355b97c..0000000000 --- a/shared/continuations/lib/PreCoroutineLowering.cpp +++ /dev/null @@ -1,195 +0,0 @@ -/* - *********************************************************************************************************************** - * - * Copyright (c) 2022-2023 Advanced Micro Devices, Inc. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - *deal in the Software without restriction, including without limitation the - *rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - *sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - *all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - *FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - *IN THE SOFTWARE. - * - **********************************************************************************************************************/ - -//===- PreCoroutineLowering.cpp - Split BB for rematerialized code --------===// -// -// A pass that splits the BB after a TraceRay/CallShader/ReportHit call. -// That moves all rematerialized code after the inlined TraceRay/etc. and -// ensures that the local root index is set before it is accessed. -// -// Also removes already inline driver functions that are not needed anymore. -// -// Also lowers the GetShaderKind() and GetCurrentFuncAddr() intrinsics which is -// now possible that driver functions have been inlined. -// -//===----------------------------------------------------------------------===// - -#include "continuations/Continuations.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringRef.h" -#include "llvm/IR/IRBuilder.h" -#include "llvm/IR/Module.h" -#include "llvm/IR/Type.h" -#include "llvm/InitializePasses.h" -#include "llvm/Transforms/Utils/BasicBlockUtils.h" -#include - -using namespace llvm; - -#define DEBUG_TYPE "pre-coroutine-lowering" - -PreCoroutineLoweringPass::PreCoroutineLoweringPass() {} - -// Split BB after _AmdRestoreSystemData. -// The coroutine passes rematerialize to the start of the basic block of a use. -// We split the block so that every rematerialized dxil intrinsic lands after -// the restore call and accesses the restored system data. -// If we did not do that, an intrinsic that is rematerialized to before -// RestoreSystemData is called gets an uninitialized system data struct as -// argument. -bool PreCoroutineLoweringPass::splitBB() { - bool Changed = false; - for (auto &F : *Mod) { - if (F.getName().startswith("_AmdRestoreSystemData")) { - for (auto &Use : make_early_inc_range(F.uses())) { - if (auto *CInst = dyn_cast(Use.getUser())) { - if (CInst->isCallee(&Use)) { - auto *Next = &*++CInst->getIterator(); - CInst->eraseFromParent(); - if (Next->isTerminator()) - continue; - SplitBlock(Next->getParent(), Next); - } - } - } - Changed = true; - } - } - return Changed; -} - -bool PreCoroutineLoweringPass::removeInlinedIntrinsics() { - bool Changed = false; - - // Remove functions - for (auto &F : make_early_inc_range(*Mod)) { - auto Name = F.getName(); - // TODO Temporarily support multiple prefixes for TraceRay - if (Name.startswith("amd.dx.TraceRay") || - Name.startswith("_cont_TraceRay") || - Name.startswith("_cont_CallShader") || - Name.startswith("_cont_ReportHit")) { - F.eraseFromParent(); - Changed = true; - } - } - return Changed; -} - -llvm::PreservedAnalyses -PreCoroutineLoweringPass::run(llvm::Module &M, - llvm::ModuleAnalysisManager &AnalysisManager) { - LLVM_DEBUG(dbgs() << "Run the pre-coroutine-lowering pass\n"); - - Mod = &M; - - bool Changed = splitBB(); - - // Remove already inlined driver functions - Changed |= removeInlinedIntrinsics(); - - Changed |= lowerGetShaderKind(); - Changed |= lowerGetCurrentFuncAddr(); - - if (Changed) - return PreservedAnalyses::none(); - return PreservedAnalyses::all(); -} - -bool PreCoroutineLoweringPass::lowerGetShaderKind() { - auto *GetShaderKind = Mod->getFunction("_AmdGetShaderKind"); - - if (!GetShaderKind) - return false; - - assert(GetShaderKind->getReturnType()->isIntegerTy(32) && - GetShaderKind->arg_size() == 0); - - if (!GetShaderKind->use_empty()) { - for (auto &Use : make_early_inc_range(GetShaderKind->uses())) { - auto *CInst = dyn_cast(Use.getUser()); - - if (!CInst || !CInst->isCallee(&Use)) { - // Non-call use. This will likely result in a remaining non-lowered use - // reported as error at the end of this function. - continue; - } - - Function *F = CInst->getFunction(); - auto Stage = lgc::rt::getLgcRtShaderStage(F); - - // Ignore GetShaderKind calls where we cannot find the shader kind. - // This happens e.g. in gpurt-implemented intrinsics that got inlined, - // but not removed. - if (!Stage) - continue; - - DXILShaderKind ShaderKind = - DXILContHelper::shaderStageToDxilShaderKind(*Stage); - auto *ShaderKindVal = ConstantInt::get(GetShaderKind->getReturnType(), - static_cast(ShaderKind)); - CInst->replaceAllUsesWith(ShaderKindVal); - CInst->eraseFromParent(); - } - } - - return true; -} - -bool PreCoroutineLoweringPass::lowerGetCurrentFuncAddr() { - auto *GetCurrentFuncAddr = Mod->getFunction("_AmdGetCurrentFuncAddr"); - - if (!GetCurrentFuncAddr) - return false; - - assert(GetCurrentFuncAddr->arg_size() == 0 && - // Returns an i32 or i64 - (GetCurrentFuncAddr->getReturnType()->isIntegerTy(32) || - GetCurrentFuncAddr->getReturnType()->isIntegerTy(64))); - - for (auto &Use : make_early_inc_range(GetCurrentFuncAddr->uses())) { - auto *CInst = dyn_cast(Use.getUser()); - - if (!CInst || !CInst->isCallee(&Use)) { - // Non-call use. This will likely result in a remaining non-lowered use - // reported as error at the end of this function. - continue; - } - - auto *FuncPtrToInt = ConstantExpr::getPtrToInt( - CInst->getFunction(), GetCurrentFuncAddr->getReturnType()); - CInst->replaceAllUsesWith(FuncPtrToInt); - CInst->eraseFromParent(); - } - - if (!GetCurrentFuncAddr->use_empty()) - report_fatal_error("Unknown uses of GetCurrentFuncAddr remain!"); - - // Delete the declaration of the intrinsic after lowering, as future calls to - // it are invalid. - GetCurrentFuncAddr->eraseFromParent(); - - return true; -} diff --git a/shared/continuations/lib/RegisterBuffer.cpp b/shared/continuations/lib/RegisterBuffer.cpp index 960a05b1b2..2164a839a6 100644 --- a/shared/continuations/lib/RegisterBuffer.cpp +++ b/shared/continuations/lib/RegisterBuffer.cpp @@ -275,43 +275,53 @@ Value *RegisterBufferPass::handleSingleLoadStore( LLVM_DEBUG(dbgs() << "register buffer: Found dynamic offset\n"); // Add a dynamic switch based on the address - auto *GlobalInt = Builder.CreatePtrToInt(Global, Builder.getInt32Ty()); - auto *AddressInt = Builder.CreatePtrToInt(Address, Builder.getInt32Ty()); - auto *Difference = Builder.CreateSub(AddressInt, GlobalInt); uint64_t RegistersByteCount = DL.getTypeStoreSize(Global->getValueType()).getFixedValue(); - Instruction *InsertI = &*Builder.GetInsertPoint(); - auto ResetInsertPoint = make_scope_exit( - [InsertI, &Builder]() { Builder.SetInsertPoint(InsertI); }); - - Instruction *Then; - Instruction *Else; - auto *Cond = - Builder.CreateICmpULT(Difference, Builder.getInt32(RegistersByteCount)); - SplitBlockAndInsertIfThenElse(Cond, InsertI, &Then, &Else); - BasicBlock *TailBB = InsertI->getParent(); - BasicBlock *ThenBB = Then->getParent(); - - // Access goes into the register part - Builder.SetInsertPoint(Then); - Instruction *ThenLoadStore = createLoadStore(Builder, Ty, StoreVal, Address, - Alignment, AATags, IsLoad); - - // Not in the register range - auto *Addr = computeMemAddr(Builder, Address); - Builder.SetInsertPoint(Else); - - Instruction *ElseLoadStore = - createLoadStore(Builder, Ty, StoreVal, Addr, Alignment, AATags, IsLoad); - if (IsLoad) { - Builder.SetInsertPoint(&*TailBB->getFirstInsertionPt()); - auto *PHI = Builder.CreatePHI(Ty, 2); - PHI->addIncoming(ThenLoadStore, ThenBB); - PHI->addIncoming(ElseLoadStore, ElseLoadStore->getParent()); - return PHI; + if (RegistersByteCount > 0) { + LLVM_DEBUG(dbgs() << "register buffer: Add dynamic switch\n"); + auto *GlobalInt = Builder.CreatePtrToInt(Global, Builder.getInt32Ty()); + auto *AddressInt = Builder.CreatePtrToInt(Address, Builder.getInt32Ty()); + auto *Difference = Builder.CreateSub(AddressInt, GlobalInt); + Instruction *InsertI = &*Builder.GetInsertPoint(); + auto ResetInsertPoint = make_scope_exit( + [InsertI, &Builder]() { Builder.SetInsertPoint(InsertI); }); + + Instruction *Then; + Instruction *Else; + auto *Cond = + Builder.CreateICmpULT(Difference, Builder.getInt32(RegistersByteCount)); + SplitBlockAndInsertIfThenElse(Cond, InsertI, &Then, &Else); + BasicBlock *TailBB = InsertI->getParent(); + BasicBlock *ThenBB = Then->getParent(); + + // Access goes into the register part + Builder.SetInsertPoint(Then); + Instruction *ThenLoadStore = createLoadStore(Builder, Ty, StoreVal, Address, + Alignment, AATags, IsLoad); + + // Not in the register range + auto *Addr = computeMemAddr(Builder, Address); + Builder.SetInsertPoint(Else); + + Instruction *ElseLoadStore = + createLoadStore(Builder, Ty, StoreVal, Addr, Alignment, AATags, IsLoad); + if (IsLoad) { + Builder.SetInsertPoint(&*TailBB->getFirstInsertionPt()); + auto *PHI = Builder.CreatePHI(Ty, 2); + PHI->addIncoming(ThenLoadStore, ThenBB); + PHI->addIncoming(ElseLoadStore, ElseLoadStore->getParent()); + return PHI; + } + return ElseLoadStore; } - return ElseLoadStore; + LLVM_DEBUG( + dbgs() << "register buffer: register-part is empty, always use memory\n"); + // RegistersByteCount is zero, so we know that the access goes into the memory + // part + auto *Addr = computeMemAddr(Builder, Address); + return createLoadStore(Builder, Ty, StoreVal, Addr, Alignment, AATags, + IsLoad); } namespace { diff --git a/shared/continuations/lib/SaveContinuationState.cpp b/shared/continuations/lib/SaveContinuationState.cpp index 622def5160..cf5787fa74 100644 --- a/shared/continuations/lib/SaveContinuationState.cpp +++ b/shared/continuations/lib/SaveContinuationState.cpp @@ -25,14 +25,14 @@ //===- SaveContinuationState.cpp - Callee-save continuation state ---------===// // -// Save and restore the overwritten global continuation state. -// -// This pass lowers the continuation.save/restore.continuation_state intrinsics. -// It also replaces all uses of continuation.getContinuationStackOffset with a +// This pass replaces all uses of continuation.getContinuationStackOffset with a // local variable and inits the stack pointer in entry functions with // continuation.initialContinuationStackPtr. -// The registerbuffer.getpointer(@CONTINUATION_STATE) calls are replaced with -// csp - ( - ). +// +// TODO: This pass used to handle a lot more regarding continuation state, +// and now only lowering of the CSP remains. The pass is now poorly named, +// and a later patch might completely remove this pass once CSP lowering +// is moved elsewhere. // //===----------------------------------------------------------------------===// @@ -49,120 +49,6 @@ using namespace llvm; #define DEBUG_TYPE "save-continuation-state" -SaveContinuationStatePass::SaveContinuationStatePass() {} - -static uint32_t getContStateSaveSize(Function *F) { - // Get entry function - auto *MD = F->getMetadata(DXILContHelper::MDContinuationName); - assert(MD && "Functions that call continuation state intrinsics need " - "continuation metadata"); - auto *MDTup = cast(MD); - auto *EntryF = mdconst::extract(MDTup->getOperand(0)); - - auto OptResult = DXILContHelper::tryGetContinuationStateByteCount(*EntryF); - assert(OptResult.has_value() && - "Continuation entry functions need continuation.state metadata"); - return OptResult.value(); -} - -static uint32_t getContStateRegisterSize(Function *F) { - uint32_t ContStateSize = getContStateSaveSize(F); - return std::min(ContStateSize, - ContinuationStateRegisterCount * RegisterBytes); -} - -static int64_t getContStateStackSize(Function *F) { - int64_t ContStateSize = getContStateSaveSize(F); - return std::max(ContStateSize - - (ContinuationStateRegisterCount * RegisterBytes), - static_cast(0)); -} - -bool SaveContinuationStatePass::lowerCalls(Function *Intr, bool IsSave) { - SmallVector ToProcess; - for (auto *U : Intr->users()) { - if (auto *Inst = dyn_cast(U)) - ToProcess.push_back(Inst); - } - - for (auto *Call : ToProcess) { - B->SetInsertPoint(Call); - auto *F = Call->getFunction(); - auto SaveSize = getContStateRegisterSize(F); - auto SaveI32s = divideCeil(SaveSize, RegisterBytes); - uint64_t NeededStackSize = - SaveI32s * (IsSave ? RegisterBytes : -RegisterBytes); - auto Csps = moveContinuationStackOffset(*B, NeededStackSize); - auto *Offset = IsSave ? Csps.first : Csps.second; - auto *I32Mem = continuationStackOffsetToPtr(*B, Offset); - - for (unsigned I = 0; I < SaveI32s; I++) { - // Copy ceil(SaveSize / RegisterSize) i32s from the global continuation - // state into scratch Increase CSP - auto *Reg = B->CreateGEP(ContState->getValueType(), ContState, - {B->getInt32(0), B->getInt32(I)}); - auto *Mem = B->CreateGEP(I32, I32Mem, B->getInt32(I)); - auto *Val = B->CreateLoad(I32, IsSave ? Reg : Mem); - B->CreateStore(Val, IsSave ? Mem : Reg); - } - - if (IsSave) { - // Add to continuation stack size metadata - DXILContHelper::addStackSize(F, NeededStackSize); - } - - Call->eraseFromParent(); - } - - return !ToProcess.empty(); -} - -bool SaveContinuationStatePass::lowerContStateGetPointer() { - bool Changed = false; - auto *CspType = getContinuationStackOffsetType(Mod->getContext()); - for (auto &F : Mod->functions()) { - if (F.getName().startswith("registerbuffer.setpointerbarrier")) { - // Remove setpointerbarrier instructions related to continuation state - for (auto &Use : make_early_inc_range(F.uses())) { - if (auto *CInst = dyn_cast(Use.getUser())) { - if (CInst->isCallee(&Use)) { - if (!isCastGlobal(ContState, CInst->getOperand(0))) - continue; - Changed = true; - CInst->eraseFromParent(); - } - } - } - } else if (F.getName().startswith("registerbuffer.getpointer")) { - // Check calls that take the continuation state as argument - for (auto &Use : make_early_inc_range(F.uses())) { - if (auto *CInst = dyn_cast(Use.getUser())) { - if (CInst->isCallee(&Use)) { - if (!isCastGlobal(ContState, CInst->getOperand(0))) - continue; - Changed = true; - - // Replace call with csp - - B->SetInsertPoint(CInst); - auto *CspOffsetPtr = - B->CreateCall(getContinuationStackOffset(*Mod)); - auto *Offset = B->CreateLoad(CspType, CspOffsetPtr); - auto *Ptr = continuationStackOffsetToPtr(*B, Offset); - int64_t SaveSize = getContStateStackSize(CInst->getFunction()); - Value *Replacement = - B->CreateGEP(B->getInt8Ty(), Ptr, B->getInt64(-SaveSize)); - Replacement = - B->CreateBitOrPointerCast(Replacement, CInst->getType()); - CInst->replaceAllUsesWith(Replacement); - CInst->eraseFromParent(); - } - } - } - } - } - return Changed; -} - void SaveContinuationStatePass::lowerCsp(Function *Intr) { DenseMap> ToProcess; for (auto *U : Intr->users()) { @@ -206,19 +92,6 @@ SaveContinuationStatePass::run(llvm::Module &M, I32 = Builder.getInt32Ty(); B = &Builder; Mod = &M; - ContState = M.getGlobalVariable(DXILContHelper::GlobalContStateName); - if (ContState) - assert(ContState->getValueType()->getArrayNumElements() == - ContinuationStateRegisterCount && - "global continuation state has an unexpected size"); - - Changed |= lowerContStateGetPointer(); - - if (auto *Intr = M.getFunction("continuation.save.continuation_state")) - Changed |= lowerCalls(Intr, true); - - if (auto *Intr = M.getFunction("continuation.restore.continuation_state")) - Changed |= lowerCalls(Intr, false); if (auto *Intr = M.getFunction("continuation.getContinuationStackOffset")) { Changed = true; diff --git a/shared/continuations/test/dx/cleanup-continuations-malloc.ll b/shared/continuations/test/dx/cleanup-continuations-malloc.ll index 86d6b3ca35..88c6a6f413 100644 --- a/shared/continuations/test/dx/cleanup-continuations-malloc.ll +++ b/shared/continuations/test/dx/cleanup-continuations-malloc.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt --verify-each -passes='lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint' -S %s 2>%t.stderr | FileCheck %s +; RUN: opt --verify-each -passes='lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" @@ -11,10 +11,9 @@ declare %continuation.token* @async_fun() define <4 x i32> @simple_await(<4 x i32> %arg) !continuation.registercount !1 { ; CHECK-LABEL: define void @simple_await( -; CHECK-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], <4 x i32> [[ARG:%.*]]) !continuation.registercount !2 !continuation !3 !continuation.state !4 !continuation.stacksize !4 { +; CHECK-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], <4 x i32> [[ARG:%.*]]) !continuation.registercount !1 !continuation !2 !continuation.state !3 !continuation.stacksize !3 { ; CHECK-NEXT: AllocaSpillBB: ; CHECK-NEXT: [[CONT_STATE:%.*]] = alloca [6 x i32], align 4 -; CHECK-NEXT: call void @continuation.save.continuation_state() ; CHECK-NEXT: [[ARG_SPILL_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; CHECK-NEXT: store <4 x i32> [[ARG]], ptr [[ARG_SPILL_ADDR]], align 4 ; CHECK-NEXT: [[RETURNADDR_SPILL_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_FRAME]], ptr [[CONT_STATE]], i32 0, i32 1 @@ -23,29 +22,34 @@ define <4 x i32> @simple_await(<4 x i32> %arg) !continuation.registercount !1 { ; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4 ; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 24 ; CHECK-NEXT: store i32 [[TMP2]], ptr [[TMP0]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP3]]) +; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP1]] to ptr addrspace(21) ; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4 -; CHECK-NEXT: store i32 [[TMP5]], ptr @CONTINUATION_STATE, align 4 -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4 -; CHECK-NEXT: store i32 [[TMP7]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 -; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4 -; CHECK-NEXT: store i32 [[TMP9]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 2), align 4 -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 -; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4 -; CHECK-NEXT: store i32 [[TMP11]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 3), align 4 -; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 -; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4 -; CHECK-NEXT: store i32 [[TMP13]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 4), align 4 -; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 -; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4 -; CHECK-NEXT: store i32 [[TMP15]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 5), align 4 -; CHECK-NEXT: [[TMP16:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4 -; CHECK-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP17]], i64 ptrtoint (ptr @simple_await.resume.0 to i64)), !continuation.registercount !2, !continuation.returnedRegistercount !2 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 0 +; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4 +; CHECK-NEXT: store i32 [[TMP6]], ptr addrspace(21) [[TMP5]], align 4 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 1 +; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4 +; CHECK-NEXT: store i32 [[TMP9]], ptr addrspace(21) [[TMP8]], align 4 +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 2 +; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP10]], align 4 +; CHECK-NEXT: store i32 [[TMP12]], ptr addrspace(21) [[TMP11]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 3 +; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP13]], align 4 +; CHECK-NEXT: store i32 [[TMP15]], ptr addrspace(21) [[TMP14]], align 4 +; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 +; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 4 +; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP16]], align 4 +; CHECK-NEXT: store i32 [[TMP18]], ptr addrspace(21) [[TMP17]], align 4 +; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 +; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 5 +; CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP19]], align 4 +; CHECK-NEXT: store i32 [[TMP21]], ptr addrspace(21) [[TMP20]], align 4 +; CHECK-NEXT: [[TMP22:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4 +; CHECK-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP23]], i64 ptrtoint (ptr @simple_await.resume.0 to i64)), !continuation.registercount !1, !continuation.returnedRegistercount !1 ; CHECK-NEXT: unreachable ; %tok = call %continuation.token* @async_fun(), !continuation.registercount !1, !continuation.returnedRegistercount !1 @@ -55,7 +59,7 @@ define <4 x i32> @simple_await(<4 x i32> %arg) !continuation.registercount !1 { define void @simple_await_entry(<4 x i32> %arg, <4 x i32> addrspace(1)* %mem) !continuation.entry !0 !continuation.registercount !1 { ; CHECK-LABEL: define void @simple_await_entry( -; CHECK-SAME: <4 x i32> [[ARG:%.*]], ptr addrspace(1) [[MEM:%.*]]) !continuation.registercount !2 !continuation.entry !5 !continuation !6 !continuation.state !4 !continuation.stacksize !4 { +; CHECK-SAME: <4 x i32> [[ARG:%.*]], ptr addrspace(1) [[MEM:%.*]]) !continuation.registercount !1 !continuation.entry !4 !continuation !5 !continuation.state !3 !continuation.stacksize !3 { ; CHECK-NEXT: AllocaSpillBB: ; CHECK-NEXT: [[CONT_STATE:%.*]] = alloca [6 x i32], align 4 ; CHECK-NEXT: [[MEM_SPILL_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_ENTRY_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 1 @@ -66,29 +70,34 @@ define void @simple_await_entry(<4 x i32> %arg, <4 x i32> addrspace(1)* %mem) !c ; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4 ; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 24 ; CHECK-NEXT: store i32 [[TMP2]], ptr [[TMP0]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP3]]) +; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP1]] to ptr addrspace(21) ; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4 -; CHECK-NEXT: store i32 [[TMP5]], ptr @CONTINUATION_STATE, align 4 -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4 -; CHECK-NEXT: store i32 [[TMP7]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 -; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4 -; CHECK-NEXT: store i32 [[TMP9]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 2), align 4 -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 -; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4 -; CHECK-NEXT: store i32 [[TMP11]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 3), align 4 -; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 -; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4 -; CHECK-NEXT: store i32 [[TMP13]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 4), align 4 -; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 -; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4 -; CHECK-NEXT: store i32 [[TMP15]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 5), align 4 -; CHECK-NEXT: [[TMP16:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4 -; CHECK-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP17]], i64 ptrtoint (ptr @simple_await_entry.resume.0 to i64)), !continuation.registercount !2, !continuation.returnedRegistercount !2 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 0 +; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4 +; CHECK-NEXT: store i32 [[TMP6]], ptr addrspace(21) [[TMP5]], align 4 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 1 +; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4 +; CHECK-NEXT: store i32 [[TMP9]], ptr addrspace(21) [[TMP8]], align 4 +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 2 +; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP10]], align 4 +; CHECK-NEXT: store i32 [[TMP12]], ptr addrspace(21) [[TMP11]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 3 +; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP13]], align 4 +; CHECK-NEXT: store i32 [[TMP15]], ptr addrspace(21) [[TMP14]], align 4 +; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 +; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 4 +; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP16]], align 4 +; CHECK-NEXT: store i32 [[TMP18]], ptr addrspace(21) [[TMP17]], align 4 +; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 +; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 5 +; CHECK-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP19]], align 4 +; CHECK-NEXT: store i32 [[TMP21]], ptr addrspace(21) [[TMP20]], align 4 +; CHECK-NEXT: [[TMP22:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4 +; CHECK-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP23]], i64 ptrtoint (ptr @simple_await_entry.resume.0 to i64)), !continuation.registercount !1, !continuation.returnedRegistercount !1 ; CHECK-NEXT: unreachable ; %tok = call %continuation.token* @async_fun(), !continuation.registercount !1, !continuation.returnedRegistercount !1 diff --git a/shared/continuations/test/dx/cleanup-continuations.ll b/shared/continuations/test/dx/cleanup-continuations.ll index 416c55bb2b..b300343f54 100644 --- a/shared/continuations/test/dx/cleanup-continuations.ll +++ b/shared/continuations/test/dx/cleanup-continuations.ll @@ -1,5 +1,5 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt --verify-each -passes='legacy-cleanup-continuations,lint' -S %s 2>%t.stderr | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --version 3 +; RUN: opt --verify-each -passes='legacy-cleanup-continuations,lint' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" @@ -13,71 +13,68 @@ declare %continuation.token* @async_fun() declare i32 @continuations.getReturnValue.i32() #0 declare void @continuation.return(i64, ...) -; CHECK: @CONTINUATION_STATE = external global [2 x i32] - -@RETURN_ADDR = external addrspace(20) global i64 - define { i8*, %continuation.token* } @simple_await(i8* %0) !continuation !0 !continuation.registercount !4 { -; CHECK-LABEL: define void @simple_await() !continuation !2 !continuation.registercount !3 !continuation.state !4 !continuation.stacksize !4 { +; CHECK-LABEL: define void @simple_await() !continuation !1 !continuation.registercount !2 !continuation.state !3 !continuation.stacksize !3 { ; CHECK-NEXT: AllocaSpillBB: ; CHECK-NEXT: [[CONT_STATE:%.*]] = alloca [2 x i32], align 4 -; CHECK-NEXT: call void @continuation.save.continuation_state() ; CHECK-NEXT: [[FRAMEPTR:%.*]] = bitcast ptr [[CONT_STATE]] to ptr -; CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr addrspace(20) @RETURN_ADDR, align 4 ; CHECK-NEXT: [[DOTSPILL_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_FRAME:%.*]], ptr [[FRAMEPTR]], i32 0, i32 0 -; CHECK-NEXT: store i64 [[TMP0]], ptr [[DOTSPILL_ADDR]], align 4 -; CHECK-NEXT: [[TMP1:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[TMP2]], 8 -; CHECK-NEXT: store i32 [[TMP3]], ptr [[TMP1]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP4]]) -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4 -; CHECK-NEXT: store i32 [[TMP6]], ptr @CONTINUATION_STATE, align 4 +; CHECK-NEXT: store i64 -1, ptr [[DOTSPILL_ADDR]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 8 +; CHECK-NEXT: store i32 [[TMP2]], ptr [[TMP0]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP1]] to ptr addrspace(21) +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 0 +; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4 +; CHECK-NEXT: store i32 [[TMP6]], ptr addrspace(21) [[TMP5]], align 4 ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4 -; CHECK-NEXT: store i32 [[TMP8]], ptr getelementptr inbounds ([2 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CHECK-NEXT: [[TMP9:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4 -; CHECK-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP10]], i64 ptrtoint (ptr @simple_await.resume.0 to i64)), !continuation.registercount !3, !continuation.returnedRegistercount !3 +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 1 +; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4 +; CHECK-NEXT: store i32 [[TMP9]], ptr addrspace(21) [[TMP8]], align 4 +; CHECK-NEXT: [[TMP10:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4 +; CHECK-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP11]], i64 ptrtoint (ptr @simple_await.resume.0 to i64)), !continuation.registercount !2, !continuation.returnedRegistercount !2 ; CHECK-NEXT: unreachable ; AllocaSpillBB: %FramePtr = bitcast i8* %0 to %simple_await.Frame* - %1 = load i64, i64 addrspace(20)* @RETURN_ADDR, align 4 %.spill.addr = getelementptr inbounds %simple_await.Frame, %simple_await.Frame* %FramePtr, i32 0, i32 0 - store i64 %1, i64* %.spill.addr, align 4 + store i64 -1, i64* %.spill.addr, align 4 %tok = call %continuation.token* @async_fun(), !continuation.registercount !4, !continuation.returnedRegistercount !4 - %2 = insertvalue { i8*, %continuation.token* } { i8* bitcast ({ i8*, %continuation.token* } (i8*, i1)* @simple_await.resume.0 to i8*), %continuation.token* undef }, %continuation.token* %tok, 1 - ret { i8*, %continuation.token* } %2 + %1 = insertvalue { i8*, %continuation.token* } { i8* bitcast ({ i8*, %continuation.token* } (i8*, i1)* @simple_await.resume.0 to i8*), %continuation.token* undef }, %continuation.token* %tok, 1 + ret { i8*, %continuation.token* } %1 } define internal { i8*, %continuation.token* } @simple_await.resume.0(i8* noalias nonnull align 16 dereferenceable(8) %0, i1 %1) !continuation !0 { -; CHECK-LABEL: define void @simple_await.resume.0( -; CHECK-SAME: i32 [[TMP0:%.*]]) !continuation !2 !continuation.registercount !3 { +; CHECK-LABEL: define dso_local void @simple_await.resume.0( +; CHECK-SAME: i32 [[TMP0:%.*]]) !continuation !1 !continuation.registercount !2 { ; CHECK-NEXT: entryresume.0: ; CHECK-NEXT: [[CONT_STATE:%.*]] = alloca [2 x i32], align 4 -; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr @CONTINUATION_STATE, align 4 -; CHECK-NEXT: store i32 [[TMP2]], ptr [[TMP1]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr getelementptr inbounds ([2 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CHECK-NEXT: store i32 [[TMP4]], ptr [[TMP3]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP5]]) -; CHECK-NEXT: [[TMP6:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4 -; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], -8 -; CHECK-NEXT: store i32 [[TMP8]], ptr [[TMP6]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(21) +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP3]], i64 -8 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 0 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(21) [[TMP5]], align 4 +; CHECK-NEXT: store i32 [[TMP7]], ptr [[TMP6]], align 4 +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 1 +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr addrspace(21) [[TMP8]], align 4 +; CHECK-NEXT: store i32 [[TMP10]], ptr [[TMP9]], align 4 +; CHECK-NEXT: [[TMP11:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[TMP12]], -8 +; CHECK-NEXT: store i32 [[TMP13]], ptr [[TMP11]], align 4 ; CHECK-NEXT: [[FRAMEPTR:%.*]] = bitcast ptr [[CONT_STATE]] to ptr ; CHECK-NEXT: [[VFRAME:%.*]] = bitcast ptr [[FRAMEPTR]] to ptr ; CHECK-NEXT: [[DOTRELOAD_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_FRAME:%.*]], ptr [[FRAMEPTR]], i32 0, i32 0 ; CHECK-NEXT: [[DOTRELOAD:%.*]] = load i64, ptr [[DOTRELOAD_ADDR]], align 4 -; CHECK-NEXT: call void @continuation.restore.continuation_state() -; CHECK-NEXT: [[TMP9:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4 -; CHECK-NEXT: call void (i64, ...) @continuation.continue(i64 [[DOTRELOAD]], i32 [[TMP10]]), !continuation.registercount !3 +; CHECK-NEXT: [[TMP14:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4 +; CHECK-NEXT: call void (i64, ...) @continuation.continue(i64 [[DOTRELOAD]], i32 [[TMP15]]), !continuation.registercount !2 ; CHECK-NEXT: unreachable ; entryresume.0: @@ -90,7 +87,7 @@ entryresume.0: } define { i8*, %continuation.token* } @simple_await_entry(i8* %0) !continuation.entry !2 !continuation !3 !continuation.registercount !4 { -; CHECK-LABEL: define void @simple_await_entry() !continuation !5 !continuation.registercount !3 !continuation.entry !6 !continuation.state !4 !continuation.stacksize !4 { +; CHECK-LABEL: define void @simple_await_entry() !continuation !4 !continuation.registercount !2 !continuation.entry !5 !continuation.state !3 !continuation.stacksize !3 { ; CHECK-NEXT: AllocaSpillBB: ; CHECK-NEXT: [[CONT_STATE:%.*]] = alloca [2 x i32], align 4 ; CHECK-NEXT: [[FRAMEPTR:%.*]] = bitcast ptr [[CONT_STATE]] to ptr @@ -98,17 +95,18 @@ define { i8*, %continuation.token* } @simple_await_entry(i8* %0) !continuation.e ; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4 ; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 8 ; CHECK-NEXT: store i32 [[TMP2]], ptr [[TMP0]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP3]]) +; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP1]] to ptr addrspace(21) ; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4 -; CHECK-NEXT: store i32 [[TMP5]], ptr @CONTINUATION_STATE, align 4 -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4 -; CHECK-NEXT: store i32 [[TMP7]], ptr getelementptr inbounds ([2 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CHECK-NEXT: [[TMP8:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4 -; CHECK-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP9]], i64 ptrtoint (ptr @simple_await_entry.resume.0 to i64)), !continuation.registercount !3, !continuation.returnedRegistercount !3 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 0 +; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4 +; CHECK-NEXT: store i32 [[TMP6]], ptr addrspace(21) [[TMP5]], align 4 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 1 +; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4 +; CHECK-NEXT: store i32 [[TMP9]], ptr addrspace(21) [[TMP8]], align 4 +; CHECK-NEXT: [[TMP10:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4 +; CHECK-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP11]], i64 ptrtoint (ptr @simple_await_entry.resume.0 to i64)), !continuation.registercount !2, !continuation.returnedRegistercount !2 ; CHECK-NEXT: unreachable ; AllocaSpillBB: @@ -121,22 +119,26 @@ AllocaSpillBB: } define internal { i8*, %continuation.token* } @simple_await_entry.resume.0(i8* noalias nonnull align 16 dereferenceable(8) %0, i1 %1) !continuation.entry !2 !continuation !3 { -; CHECK-LABEL: define void @simple_await_entry.resume.0( -; CHECK-SAME: i32 [[TMP0:%.*]]) !continuation !5 !continuation.registercount !3 { +; CHECK-LABEL: define dso_local void @simple_await_entry.resume.0( +; CHECK-SAME: i32 [[TMP0:%.*]]) !continuation !4 !continuation.registercount !2 { ; CHECK-NEXT: entryresume.0: ; CHECK-NEXT: [[CONT_STATE:%.*]] = alloca [2 x i32], align 4 -; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr @CONTINUATION_STATE, align 4 -; CHECK-NEXT: store i32 [[TMP2]], ptr [[TMP1]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr getelementptr inbounds ([2 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CHECK-NEXT: store i32 [[TMP4]], ptr [[TMP3]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP5]]) -; CHECK-NEXT: [[TMP6:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4 -; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], -8 -; CHECK-NEXT: store i32 [[TMP8]], ptr [[TMP6]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(21) +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP3]], i64 -8 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 0 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(21) [[TMP5]], align 4 +; CHECK-NEXT: store i32 [[TMP7]], ptr [[TMP6]], align 4 +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 1 +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr addrspace(21) [[TMP8]], align 4 +; CHECK-NEXT: store i32 [[TMP10]], ptr [[TMP9]], align 4 +; CHECK-NEXT: [[TMP11:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[TMP12]], -8 +; CHECK-NEXT: store i32 [[TMP13]], ptr [[TMP11]], align 4 ; CHECK-NEXT: [[FRAMEPTR:%.*]] = bitcast ptr [[CONT_STATE]] to ptr ; CHECK-NEXT: [[VFRAME:%.*]] = bitcast ptr [[FRAMEPTR]] to ptr ; CHECK-NEXT: call void @continuation.complete() @@ -150,63 +152,64 @@ entryresume.0: } define { i8*, %continuation.token* } @await_with_ret_value(i8* %0) !continuation !1 !continuation.registercount !4 { -; CHECK-LABEL: define void @await_with_ret_value() !continuation !7 !continuation.registercount !3 !continuation.state !4 !continuation.stacksize !4 { +; CHECK-LABEL: define void @await_with_ret_value() !continuation !6 !continuation.registercount !2 !continuation.state !3 !continuation.stacksize !3 { ; CHECK-NEXT: [[CONT_STATE:%.*]] = alloca [2 x i32], align 4 -; CHECK-NEXT: call void @continuation.save.continuation_state() ; CHECK-NEXT: [[FRAMEPTR:%.*]] = bitcast ptr [[CONT_STATE]] to ptr -; CHECK-NEXT: [[RETADDR:%.*]] = load i64, ptr addrspace(20) @RETURN_ADDR, align 4 ; CHECK-NEXT: [[DOTSPILL_ADDR:%.*]] = getelementptr inbounds [[AWAIT_WITH_RET_VALUE_FRAME:%.*]], ptr [[FRAMEPTR]], i32 0, i32 0 -; CHECK-NEXT: store i64 [[RETADDR]], ptr [[DOTSPILL_ADDR]], align 4 +; CHECK-NEXT: store i64 -1, ptr [[DOTSPILL_ADDR]], align 4 ; CHECK-NEXT: [[TMP1:%.*]] = call ptr @continuation.getContinuationStackOffset() ; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4 ; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[TMP2]], 8 ; CHECK-NEXT: store i32 [[TMP3]], ptr [[TMP1]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP4]]) +; CHECK-NEXT: [[TMP4:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(21) ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4 -; CHECK-NEXT: store i32 [[TMP6]], ptr @CONTINUATION_STATE, align 4 -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4 -; CHECK-NEXT: store i32 [[TMP8]], ptr getelementptr inbounds ([2 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CHECK-NEXT: [[TMP9:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4 -; CHECK-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP10]], i64 ptrtoint (ptr @await_with_ret_value.resume.0 to i64)), !continuation.registercount !3, !continuation.returnedRegistercount !3 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 0 +; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP5]], align 4 +; CHECK-NEXT: store i32 [[TMP7]], ptr addrspace(21) [[TMP6]], align 4 +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 1 +; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP8]], align 4 +; CHECK-NEXT: store i32 [[TMP10]], ptr addrspace(21) [[TMP9]], align 4 +; CHECK-NEXT: [[TMP11:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4 +; CHECK-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP12]], i64 ptrtoint (ptr @await_with_ret_value.resume.0 to i64)), !continuation.registercount !2, !continuation.returnedRegistercount !2 ; CHECK-NEXT: unreachable ; %FramePtr = bitcast i8* %0 to %await_with_ret_value.Frame* - %retaddr = load i64, i64 addrspace(20)* @RETURN_ADDR, align 4 %.spill.addr = getelementptr inbounds %await_with_ret_value.Frame, %await_with_ret_value.Frame* %FramePtr, i32 0, i32 0 - store i64 %retaddr, i64* %.spill.addr, align 4 + store i64 -1, i64* %.spill.addr, align 4 %tok = call %continuation.token* @async_fun(), !continuation.registercount !4, !continuation.returnedRegistercount !4 %res = insertvalue { i8*, %continuation.token* } { i8* bitcast ({ i8*, %continuation.token* } (i8*, i1)* @await_with_ret_value.resume.0 to i8*), %continuation.token* undef }, %continuation.token* %tok, 1 ret { i8*, %continuation.token* } %res } define internal { i8*, %continuation.token* } @await_with_ret_value.resume.0(i8* noalias nonnull align 16 dereferenceable(8) %0, i1 %1) !continuation !1 { -; CHECK-LABEL: define void @await_with_ret_value.resume.0( -; CHECK-SAME: i32 [[TMP0:%.*]], i32 [[RES1:%.*]]) !continuation !7 !continuation.registercount !3 { +; CHECK-LABEL: define dso_local void @await_with_ret_value.resume.0( +; CHECK-SAME: i32 [[TMP0:%.*]], i32 [[RES1:%.*]]) !continuation !6 !continuation.registercount !2 { ; CHECK-NEXT: [[CONT_STATE:%.*]] = alloca [2 x i32], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr @CONTINUATION_STATE, align 4 -; CHECK-NEXT: store i32 [[TMP3]], ptr [[TMP2]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr getelementptr inbounds ([2 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CHECK-NEXT: store i32 [[TMP5]], ptr [[TMP4]], align 4 -; CHECK-NEXT: [[TMP6:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP6]]) -; CHECK-NEXT: [[TMP7:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4 -; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[TMP8]], -8 -; CHECK-NEXT: store i32 [[TMP9]], ptr [[TMP7]], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4 +; CHECK-NEXT: [[TMP4:%.*]] = inttoptr i32 [[TMP3]] to ptr addrspace(21) +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP4]], i64 -8 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP5]], i32 0, i32 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr addrspace(21) [[TMP6]], align 4 +; CHECK-NEXT: store i32 [[TMP8]], ptr [[TMP7]], align 4 +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP5]], i32 0, i32 1 +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr addrspace(21) [[TMP9]], align 4 +; CHECK-NEXT: store i32 [[TMP11]], ptr [[TMP10]], align 4 +; CHECK-NEXT: [[TMP12:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4 +; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], -8 +; CHECK-NEXT: store i32 [[TMP14]], ptr [[TMP12]], align 4 ; CHECK-NEXT: [[FRAMEPTR:%.*]] = bitcast ptr [[CONT_STATE]] to ptr ; CHECK-NEXT: [[VFRAME:%.*]] = bitcast ptr [[FRAMEPTR]] to ptr ; CHECK-NEXT: [[DOTRELOAD_ADDR:%.*]] = getelementptr inbounds [[AWAIT_WITH_RET_VALUE_FRAME:%.*]], ptr [[FRAMEPTR]], i32 0, i32 0 ; CHECK-NEXT: [[DOTRELOAD:%.*]] = load i64, ptr [[DOTRELOAD_ADDR]], align 4 -; CHECK-NEXT: call void @continuation.restore.continuation_state() -; CHECK-NEXT: [[TMP10:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4 -; CHECK-NEXT: call void (i64, ...) @continuation.continue(i64 [[DOTRELOAD]], i32 [[TMP11]], i32 [[RES1]]), !continuation.registercount !3 +; CHECK-NEXT: [[TMP15:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4 +; CHECK-NEXT: call void (i64, ...) @continuation.continue(i64 [[DOTRELOAD]], i32 [[TMP16]], i32 [[RES1]]), !continuation.registercount !2 ; CHECK-NEXT: unreachable ; %FramePtr = bitcast i8* %0 to %await_with_ret_value.Frame* @@ -228,3 +231,16 @@ attributes #0 = { nounwind } !3 = !{{ i8*, %continuation.token* } (i8*)* @simple_await_entry} !4 = !{i32 0} !5 = !{i32 21} +;. +; CHECK: attributes #[[ATTR0:[0-9]+]] = { nounwind } +; CHECK: attributes #[[ATTR1:[0-9]+]] = { noreturn } +; CHECK: attributes #[[ATTR2:[0-9]+]] = { nofree norecurse nosync nounwind speculatable willreturn memory(none) } +;. +; CHECK: [[META0:![0-9]+]] = !{i32 21} +; CHECK: [[META1:![0-9]+]] = !{ptr @simple_await} +; CHECK: [[META2:![0-9]+]] = !{i32 0} +; CHECK: [[META3:![0-9]+]] = !{i32 8} +; CHECK: [[META4:![0-9]+]] = !{ptr @simple_await_entry} +; CHECK: [[META5:![0-9]+]] = !{} +; CHECK: [[META6:![0-9]+]] = !{ptr @await_with_ret_value} +;. diff --git a/shared/continuations/test/dx/closest-hit-procedural.ll b/shared/continuations/test/dx/closest-hit-procedural.ll index 7f00352d5c..ec2e984a67 100644 --- a/shared/continuations/test/dx/closest-hit-procedural.ll +++ b/shared/continuations/test/dx/closest-hit-procedural.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2>%t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2>%t1.stderr | FileCheck -check-prefix=DXILCONTPOSTPROCESS %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2> %t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2> %t1.stderr | FileCheck -check-prefix=DXILCONTPOSTPROCESS %s ; RUN: count 0 < %t1.stderr ; Check a procedural closest hit shader with hit attributes that does not fit into system data alone @@ -56,15 +56,6 @@ define i32 @_cont_GetLocalRootIndex(%struct.DispatchSystemData* %data) #0 !types } define void @_cont_TraceRay(%struct.DispatchSystemData* %data, i64 %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, float %6, float %7, float %8, float %9, float %10, float %11, float %12, float %13) #0 !types !25 { -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_TraceRay( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i64 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]], i32 [[TMP3:%.*]], i32 [[TMP4:%.*]], i32 [[TMP5:%.*]], float [[TMP6:%.*]], float [[TMP7:%.*]], float [[TMP8:%.*]], float [[TMP9:%.*]], float [[TMP10:%.*]], float [[TMP11:%.*]], float [[TMP12:%.*]], float [[TMP13:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @_cont_Traversal([[STRUCT_TRAVERSALDATA]] [[TRAV_DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP15]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; %dis_data = load %struct.DispatchSystemData, %struct.DispatchSystemData* %data, align 4 %sys_data = insertvalue %struct.SystemData undef, %struct.DispatchSystemData %dis_data, 0 %trav_data = insertvalue %struct.TraversalData undef, %struct.SystemData %sys_data, 0 @@ -74,13 +65,6 @@ define void @_cont_TraceRay(%struct.DispatchSystemData* %data, i64 %0, i32 %1, i } define i1 @_cont_ReportHit(%struct.AnyHitTraversalData* %data, float %t, i32 %hitKind) #0 !types !26 { -; LOWERRAYTRACINGPIPELINE-LABEL: define i1 @_cont_ReportHit( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], float [[T:%.*]], i32 [[HITKIND:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP1:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @_AmdAwaitAnyHit(i64 3, [[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA]], float [[T]], i32 [[HITKIND]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP1]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret i1 true -; %trav_data = load %struct.AnyHitTraversalData, %struct.AnyHitTraversalData* %data, align 4 %newdata = call %struct.AnyHitTraversalData @_AmdAwaitAnyHit(i64 3, %struct.AnyHitTraversalData %trav_data, float %t, i32 %hitKind) store %struct.AnyHitTraversalData %newdata, %struct.AnyHitTraversalData* %data, align 4 @@ -135,7 +119,7 @@ declare !types !35 i32 @_cont_HitKind(%struct.SystemData* nocapture readnone, %s ; Function Attrs: nounwind define void @ClosestHit(%struct.RayPayload* noalias nocapture %payload, %struct.HitAttributes* nocapture readonly %attr) #3 !types !36 { ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.DispatchSystemData @ClosestHit( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR4:[0-9]+]] !continuation !18 !lgc.rt.shaderstage !19 !continuation.registercount !20 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR4:[0-9]+]] !lgc.rt.shaderstage [[META19:![0-9]+]] !continuation [[META20:![0-9]+]] !continuation.registercount [[META21:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_RAYPAYLOAD:%.*]], align 8 @@ -143,8 +127,7 @@ define void @ClosestHit(%struct.RayPayload* noalias nocapture %payload, %struct. ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = call [[STRUCT_SYSTEMDATA]] @continuations.getSystemData.s_struct.SystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_SYSTEMDATA]] [[TMP4]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP3]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[TMP7]], i64 0 @@ -195,10 +178,10 @@ define void @ClosestHit(%struct.RayPayload* noalias nocapture %payload, %struct. ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP38]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP40:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP39]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP40]], !continuation.registercount !20 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP40]], !continuation.registercount [[META21]] ; ; DXILCONTPOSTPROCESS-LABEL: define void @ClosestHit( -; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR1:[0-9]+]] !continuation !17 !lgc.rt.shaderstage !18 !continuation.registercount !19 !continuation.state !14 { +; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR1:[0-9]+]] !lgc.rt.shaderstage [[META17:![0-9]+]] !continuation [[META18:![0-9]+]] !continuation.registercount [[META19:![0-9]+]] !continuation.state [[META14:![0-9]+]] { ; DXILCONTPOSTPROCESS-NEXT: AllocaSpillBB: ; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 ; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 @@ -233,7 +216,7 @@ define void @ClosestHit(%struct.RayPayload* noalias nocapture %payload, %struct. ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_LOAD:%.*]] = load <3 x i32>, ptr [[DOTFCA_0_GEP]], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, <3 x i32> [[DOTFCA_0_LOAD]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[TMP13:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP13]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount !19 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP13]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount [[META19]] ; DXILCONTPOSTPROCESS-NEXT: unreachable ; ret void diff --git a/shared/continuations/test/dx/closest-hit-traceray.ll b/shared/continuations/test/dx/closest-hit-traceray.ll index bd3d58172e..d86cd10de8 100644 --- a/shared/continuations/test/dx/closest-hit-traceray.ll +++ b/shared/continuations/test/dx/closest-hit-traceray.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2>%t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2> %t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s ; RUN: count 0 < %t0.stderr -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2>%t1.stderr | FileCheck -check-prefix=DXILCONTPOSTPROCESS %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2> %t1.stderr | FileCheck -check-prefix=DXILCONTPOSTPROCESS %s ; RUN: count 0 < %t1.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" @@ -54,15 +54,6 @@ define i32 @_cont_GetLocalRootIndex(%struct.DispatchSystemData* %data) #0 !types } define void @_cont_TraceRay(%struct.DispatchSystemData* %data, i64 %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, float %6, float %7, float %8, float %9, float %10, float %11, float %12, float %13) #0 !types !25 { -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_TraceRay( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i64 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]], i32 [[TMP3:%.*]], i32 [[TMP4:%.*]], i32 [[TMP5:%.*]], float [[TMP6:%.*]], float [[TMP7:%.*]], float [[TMP8:%.*]], float [[TMP9:%.*]], float [[TMP10:%.*]], float [[TMP11:%.*]], float [[TMP12:%.*]], float [[TMP13:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @amd.dx.Traversal([[STRUCT_TRAVERSALDATA]] [[TRAV_DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP15]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; %dis_data = load %struct.DispatchSystemData, %struct.DispatchSystemData* %data, align 4 %sys_data = insertvalue %struct.SystemData undef, %struct.DispatchSystemData %dis_data, 0 %trav_data = insertvalue %struct.TraversalData undef, %struct.SystemData %sys_data, 0 @@ -72,13 +63,6 @@ define void @_cont_TraceRay(%struct.DispatchSystemData* %data, i64 %0, i32 %1, i } define i1 @_cont_ReportHit(%struct.AnyHitTraversalData* %data, float %t, i32 %hitKind) #0 !types !26 { -; LOWERRAYTRACINGPIPELINE-LABEL: define i1 @_cont_ReportHit( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], float [[T:%.*]], i32 [[HITKIND:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP1:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @_AmdAwaitAnyHit(i64 3, [[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA]], float [[T]], i32 [[HITKIND]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP1]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret i1 true -; %trav_data = load %struct.AnyHitTraversalData, %struct.AnyHitTraversalData* %data, align 4 %newdata = call %struct.AnyHitTraversalData @_AmdAwaitAnyHit(i64 3, %struct.AnyHitTraversalData %trav_data, float %t, i32 %hitKind) store %struct.AnyHitTraversalData %newdata, %struct.AnyHitTraversalData* %data, align 4 @@ -133,7 +117,7 @@ declare !types !35 i32 @_cont_HitKind(%struct.SystemData* nocapture readnone, %s ; Function Attrs: nounwind define void @ClosestHit(%struct.RayPayload* noalias nocapture %payload, %struct.BuiltInTriangleIntersectionAttributes* nocapture readonly %attr) #3 !types !36 { ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.DispatchSystemData @ClosestHit( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR4:[0-9]+]] !continuation !18 !lgc.rt.shaderstage !19 !continuation.registercount !20 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR4:[0-9]+]] !continuation [[META18:![0-9]+]] !lgc.rt.shaderstage [[META19:![0-9]+]] !continuation.registercount [[META20:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_RAYPAYLOAD:%.*]], align 8 @@ -141,8 +125,7 @@ define void @ClosestHit(%struct.RayPayload* noalias nocapture %payload, %struct. ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = call [[STRUCT_SYSTEMDATA]] @continuations.getSystemData.s_struct.SystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_SYSTEMDATA]] [[TMP4]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP3]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[TMP7]], i64 0 @@ -180,29 +163,33 @@ define void @ClosestHit(%struct.RayPayload* noalias nocapture %payload, %struct. ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP30:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP29]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP31:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[TMP30]]) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP32:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_TraceRay.struct.RayPayload.attr_max_32_bytes(ptr [[TMP32]], i64 [[TMP31]], i32 16, i32 -1, i32 0, i32 1, i32 0, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0x3F50624DE0000000, float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 1.000000e+04, ptr [[TMP26]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA_I:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP32]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA_I:%.*]] = insertvalue [[STRUCT_SYSTEMDATA]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I]], 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA_I:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA_I]], 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP33:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @amd.dx.Traversal([[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP33]], ptr [[TMP32]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP33:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP3]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = getelementptr i32, ptr [[TMP33]], i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP35:%.*]] = getelementptr i32, ptr [[TMP34]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP36:%.*]] = load i32, ptr [[TMP35]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP36]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP37:%.*]] = getelementptr i32, ptr [[TMP33]], i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = getelementptr i32, ptr [[TMP37]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = load i32, ptr [[TMP38]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP39]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP40:%.*]] = getelementptr i32, ptr [[TMP37]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP41:%.*]] = load i32, ptr [[TMP40]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP41]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP42:%.*]] = getelementptr i32, ptr [[TMP37]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP43:%.*]] = load i32, ptr [[TMP42]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP43]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP44:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP45:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP44]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP45]], !continuation.registercount !20 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP3]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP35:%.*]] = getelementptr i32, ptr [[TMP34]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP36:%.*]] = getelementptr i32, ptr [[TMP35]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP37:%.*]] = load i32, ptr [[TMP36]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP37]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = getelementptr i32, ptr [[TMP34]], i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = getelementptr i32, ptr [[TMP38]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP40:%.*]] = load i32, ptr [[TMP39]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP40]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP41:%.*]] = getelementptr i32, ptr [[TMP38]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP42:%.*]] = load i32, ptr [[TMP41]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP42]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP43:%.*]] = getelementptr i32, ptr [[TMP38]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP44:%.*]] = load i32, ptr [[TMP43]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP44]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP45:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP46:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[TMP45]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP46]], !continuation.registercount [[META20]] ; ; DXILCONTPOSTPROCESS-LABEL: define void @ClosestHit( -; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR1:[0-9]+]] !continuation !17 !lgc.rt.shaderstage !18 !continuation.registercount !19 !continuation.state !14 { +; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR1:[0-9]+]] !continuation [[META17:![0-9]+]] !lgc.rt.shaderstage [[META18:![0-9]+]] !continuation.registercount [[META19:![0-9]+]] !continuation.state [[META14:![0-9]+]] { ; DXILCONTPOSTPROCESS-NEXT: AllocaSpillBB: ; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 ; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 @@ -250,7 +237,7 @@ define void @ClosestHit(%struct.RayPayload* noalias nocapture %payload, %struct. ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_LOAD:%.*]] = load <3 x i32>, ptr [[DOTFCA_0_GEP]], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, <3 x i32> [[DOTFCA_0_LOAD]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[TMP18:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP18]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount !19 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP18]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount [[META19]] ; DXILCONTPOSTPROCESS-NEXT: unreachable ; %1 = load %dx.types.Handle, %dx.types.Handle* @"\01?Scene@@3URaytracingAccelerationStructure@@A", align 4 diff --git a/shared/continuations/test/dx/closest-hit.ll b/shared/continuations/test/dx/closest-hit.ll index f759b99e1a..af8d138fc9 100644 --- a/shared/continuations/test/dx/closest-hit.ll +++ b/shared/continuations/test/dx/closest-hit.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2>%t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2> %t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s ; RUN: count 0 < %t0.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" @@ -41,15 +41,6 @@ define i32 @_cont_GetLocalRootIndex(%struct.DispatchSystemData* %data) #0 !types } define void @_cont_TraceRay(%struct.DispatchSystemData* %data, i64 %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, float %6, float %7, float %8, float %9, float %10, float %11, float %12, float %13) #0 !types !19 { -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_TraceRay( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i64 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]], i32 [[TMP3:%.*]], i32 [[TMP4:%.*]], i32 [[TMP5:%.*]], float [[TMP6:%.*]], float [[TMP7:%.*]], float [[TMP8:%.*]], float [[TMP9:%.*]], float [[TMP10:%.*]], float [[TMP11:%.*]], float [[TMP12:%.*]], float [[TMP13:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @_cont_Traversal([[STRUCT_TRAVERSALDATA]] [[TRAV_DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP15]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; %dis_data = load %struct.DispatchSystemData, %struct.DispatchSystemData* %data, align 4 %sys_data = insertvalue %struct.SystemData undef, %struct.DispatchSystemData %dis_data, 0 %trav_data = insertvalue %struct.TraversalData undef, %struct.SystemData %sys_data, 0 @@ -59,13 +50,6 @@ define void @_cont_TraceRay(%struct.DispatchSystemData* %data, i64 %0, i32 %1, i } define i1 @_cont_ReportHit(%struct.AnyHitTraversalData* %data, float %t, i32 %hitKind) #0 !types !20 { -; LOWERRAYTRACINGPIPELINE-LABEL: define i1 @_cont_ReportHit( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], float [[T:%.*]], i32 [[HITKIND:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP1:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @_AmdAwaitAnyHit(i64 3, [[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA]], float [[T]], i32 [[HITKIND]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP1]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret i1 true -; %trav_data = load %struct.AnyHitTraversalData, %struct.AnyHitTraversalData* %data, align 4 %newdata = call %struct.AnyHitTraversalData @_AmdAwaitAnyHit(i64 3, %struct.AnyHitTraversalData %trav_data, float %t, i32 %hitKind) store %struct.AnyHitTraversalData %newdata, %struct.AnyHitTraversalData* %data, align 4 @@ -120,7 +104,7 @@ declare !types !29 i32 @_cont_HitKind(%struct.SystemData* nocapture readnone, %s ; Function Attrs: nounwind define void @ClosestHit(%struct.RayPayload* noalias nocapture %payload, %struct.BuiltInTriangleIntersectionAttributes* nocapture readonly %attr) #3 !types !30 { ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.DispatchSystemData @ClosestHit( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR4:[0-9]+]] !continuation !12 !lgc.rt.shaderstage !13 !continuation.registercount !14 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR4:[0-9]+]] !lgc.rt.shaderstage [[META13:![0-9]+]] !continuation [[META14:![0-9]+]] !continuation.registercount [[META15:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_RAYPAYLOAD:%.*]], align 8 @@ -128,8 +112,7 @@ define void @ClosestHit(%struct.RayPayload* noalias nocapture %payload, %struct. ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = call [[STRUCT_SYSTEMDATA]] @continuations.getSystemData.s_struct.SystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_SYSTEMDATA]] [[TMP4]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP3]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[TMP7]], i64 0 @@ -166,7 +149,7 @@ define void @ClosestHit(%struct.RayPayload* noalias nocapture %payload, %struct. ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP26]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP27:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP27]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP28]], !continuation.registercount !14 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP28]], !continuation.registercount [[META15]] ; %ptr = getelementptr inbounds %struct.RayPayload, %struct.RayPayload* %payload, i32 0, i32 0 %baryPtr = getelementptr inbounds %struct.BuiltInTriangleIntersectionAttributes, %struct.BuiltInTriangleIntersectionAttributes* %attr, i32 0, i32 0 diff --git a/shared/continuations/test/dx/continuation-registercount.ll b/shared/continuations/test/dx/continuation-registercount.ll index 5bcc264f14..23be659256 100644 --- a/shared/continuations/test/dx/continuation-registercount.ll +++ b/shared/continuations/test/dx/continuation-registercount.ll @@ -1,14 +1,14 @@ ; RUN: grep -v SKIP_LINE_BY_DEFAULT %s | \ -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S 2>%t0.stderr | \ +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S 2> %t0.stderr | \ ; RUN: FileCheck -check-prefix=POSTPROCESS-REGCOUNT %s ; RUN: count 0 < %t0.stderr ; ; RUN: grep -v SKIP_LINE_BY_DEFAULT %s | \ -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S 2>%t1.stderr | \ +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S 2> %t1.stderr | \ ; RUN: FileCheck -check-prefix=POSTPROCESS-REGCOUNT2 %s ; RUN: count 0 < %t1.stderr ; -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2>%t2.stderr | \ +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2> %t2.stderr | \ ; RUN: FileCheck -check-prefix=POSTPROCESS-REGCOUNT-FEWREGS %s ; RUN: count 0 < %t2.stderr @@ -145,14 +145,14 @@ define void @mainTrace() { } ; POSTPROCESS-REGCOUNT-DAG: define void @called({{.*}}%struct.DispatchSystemData %0){{.*}} !continuation.registercount ![[called_registercount:[0-9]+]] -; POSTPROCESS-REGCOUNT-DAG: define void @called.resume.0({{.*}}%struct.DispatchSystemData %1){{.*}} !continuation.registercount ![[called_resume_registercount:[0-9]+]] +; POSTPROCESS-REGCOUNT-DAG: define dso_local void @called.resume.0({{.*}}%struct.DispatchSystemData %1){{.*}} !continuation.registercount ![[called_resume_registercount:[0-9]+]] ; POSTPROCESS-REGCOUNT-DAG: ![[called_registercount]] = !{i32 26} ; POSTPROCESS-REGCOUNT-DAG: ![[called_resume_registercount]] = !{i32 27} ; If we set maxPayloadRegisterCount to 10, both functions use only 10 payload registers. ; Note that due to metadata uniquing, both use the same metadata node. ; POSTPROCESS-REGCOUNT-FEWREGS-DAG: define void @called({{.*}}%struct.DispatchSystemData %0){{.*}} !continuation.registercount ![[registercount:[0-9]+]] -; POSTPROCESS-REGCOUNT-FEWREGS-DAG: define void @called.resume.0({{.*}}%struct.DispatchSystemData %1){{.*}} !continuation.registercount ![[registercount]] +; POSTPROCESS-REGCOUNT-FEWREGS-DAG: define dso_local void @called.resume.0({{.*}}%struct.DispatchSystemData %1){{.*}} !continuation.registercount ![[registercount]] ; POSTPROCESS-REGCOUNT-FEWREGS-DAG: ![[registercount]] = !{i32 10} define void @called(%struct.MyParams* %arg) !types !39 { @@ -162,7 +162,7 @@ define void @called(%struct.MyParams* %arg) !types !39 { } ; POSTPROCESS-REGCOUNT-DAG: define void @Intersection({{.*}}%struct.AnyHitTraversalData %0){{.*}} !continuation.registercount ![[intersection_registercount:[0-9]+]] -; POSTPROCESS-REGCOUNT-DAG: define void @Intersection.resume.0({{.*}}%struct.AnyHitTraversalData %1){{.*}} !continuation.registercount ![[intersection_registercount]] +; POSTPROCESS-REGCOUNT-DAG: define dso_local void @Intersection.resume.0({{.*}}%struct.AnyHitTraversalData %1){{.*}} !continuation.registercount ![[intersection_registercount]] ; POSTPROCESS-REGCOUNT-DAG: call void (i64, ...) @continuation.continue(i64 3, {{.*}} float 4.000000e+00, i32 0, %struct.BuiltInTriangleIntersectionAttributes {{.*}}), !continuation.registercount ![[intersection_registercount]] ; POSTPROCESS-REGCOUNT-DAG: ![[intersection_registercount]] = !{i32 30} @@ -189,14 +189,14 @@ define void @ClosestHit(%struct.RayPayload* noalias nocapture %payload, %struct. declare void @continuation.continue(i64, ...) -; POSTPROCESS-REGCOUNT-FEWREGS-DAG: define void @TraversalImpl1_1({{.*}} !continuation.registercount ![[registercount]] +; POSTPROCESS-REGCOUNT-FEWREGS-DAG: define {{.*}} @_cont_Traversal({{.*}} !continuation.registercount ![[registercount]] ; ^--- this MD node has value 10 ; POSTPROCESS-REGCOUNT-FEWREGS-DAG: call {{.*}} @continuation.continue({{.*}} !continuation.registercount ![[registercount]] -; POSTPROCESS-REGCOUNT-DAG: define void @TraversalImpl1_1({{.*}} !continuation.registercount ![[intersection_registercount]] +; POSTPROCESS-REGCOUNT-DAG: define {{.*}} @_cont_Traversal({{.*}} !continuation.registercount ![[intersection_registercount]] ; ^--- this MD node has value 30 ; POSTPROCESS-REGCOUNT-DAG: call {{.*}} @continuation.continue({{.*}} !continuation.registercount ![[intersection_registercount]] -define void @TraversalImpl1_1(%struct._AmdTraversalResultData* noalias nocapture sret(%struct._AmdTraversalResultData) %agg.result, i32 %csp, %struct._AmdSystemData* noalias %data) !types !44 { +define void @_cont_Traversal(%struct._AmdTraversalResultData* noalias nocapture sret(%struct._AmdTraversalResultData) %agg.result, i32 %csp, %struct._AmdSystemData* noalias %data) !types !44 { call void (i64, ...) @continuation.continue(i64 0, i8 addrspace(21)* undef) ret void } diff --git a/shared/continuations/test/dx/continuation-stacksize.ll b/shared/continuations/test/dx/continuation-stacksize.ll index 9cce0c693f..7b88f84b92 100644 --- a/shared/continuations/test/dx/continuation-stacksize.ll +++ b/shared/continuations/test/dx/continuation-stacksize.ll @@ -1,18 +1,18 @@ -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2>%t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE-STACKSIZE %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2> %t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE-STACKSIZE %s ; RUN: count 0 < %t0.stderr -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,remove-types-metadata' \ -; RUN: -S %s 2>%t1.stderr | FileCheck -check-prefix=CLEANUP-STACKSIZE %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,remove-types-metadata' \ +; RUN: -S %s 2> %t1.stderr | FileCheck -check-prefix=CLEANUP-STACKSIZE %s ; RUN: count 0 < %t1.stderr -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,remove-types-metadata' \ -; RUN: -S %s 2>%t2.stderr | FileCheck -check-prefix=CLEANUP-STATESIZE %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,remove-types-metadata' \ +; RUN: -S %s 2> %t2.stderr | FileCheck -check-prefix=CLEANUP-STATESIZE %s ; RUN: count 0 < %t2.stderr -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,remove-types-metadata' \ -; RUN: -S %s 2>%t3.stderr | FileCheck -check-prefix=SAVESTATE-STACKSIZE %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,remove-types-metadata' \ +; RUN: -S %s 2> %t3.stderr | FileCheck -check-prefix=SAVESTATE-STACKSIZE %s ; RUN: count 0 < %t3.stderr -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,remove-types-metadata' \ -; RUN: -S %s 2>%t4.stderr | FileCheck -check-prefix=SAVESTATE-STATESIZE %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,remove-types-metadata' \ +; RUN: -S %s 2> %t4.stderr | FileCheck -check-prefix=SAVESTATE-STATESIZE %s ; RUN: count 0 < %t4.stderr ; The order of metadata on functions is non-deterministic, so make two different runs to match both of them. diff --git a/shared/continuations/test/dx/continuation-state.ll b/shared/continuations/test/dx/continuation-state.ll index 445aa738b1..616bfddb69 100644 --- a/shared/continuations/test/dx/continuation-state.ll +++ b/shared/continuations/test/dx/continuation-state.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 3 -; RUN: opt --verify-each -passes='lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint' -S %s 2>%t0.stderr | FileCheck -check-prefix=CLEANUP %s +; RUN: opt --verify-each -passes='lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint' -S %s 2> %t0.stderr | FileCheck -check-prefix=CLEANUP %s ; RUN: count 0 < %t0.stderr ; RUN: opt --verify-each -passes='lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint' \ -; RUN: -S %s 2>%t1.stderr | FileCheck -check-prefix=POST-PROCESS %s +; RUN: -S %s 2> %t1.stderr | FileCheck -check-prefix=POST-PROCESS %s ; RUN: count 0 < %t1.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" @@ -36,10 +36,9 @@ define void @simple_await_entry(<4 x i32> %arg, <4 x i32> addrspace(1)* %mem) !c !2 = !{i32 30} !3 = !{i32 21} ; CLEANUP-LABEL: define void @simple_await( -; CLEANUP-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], <4 x i32> [[ARG:%.*]]) !continuation.registercount !3 !continuation !4 !continuation.state !5 !continuation.stacksize !5 { +; CLEANUP-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], <4 x i32> [[ARG:%.*]]) !continuation.registercount !2 !continuation !3 !continuation.state !4 !continuation.stacksize !4 { ; CLEANUP-NEXT: AllocaSpillBB: ; CLEANUP-NEXT: [[CONT_STATE:%.*]] = alloca [6 x i32], align 4 -; CLEANUP-NEXT: call void @continuation.save.continuation_state() ; CLEANUP-NEXT: [[ARG_SPILL_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; CLEANUP-NEXT: store <4 x i32> [[ARG]], ptr [[ARG_SPILL_ADDR]], align 4 ; CLEANUP-NEXT: [[RETURNADDR_SPILL_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_FRAME]], ptr [[CONT_STATE]], i32 0, i32 1 @@ -48,73 +47,85 @@ define void @simple_await_entry(<4 x i32> %arg, <4 x i32> addrspace(1)* %mem) !c ; CLEANUP-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4 ; CLEANUP-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 24 ; CLEANUP-NEXT: store i32 [[TMP2]], ptr [[TMP0]], align 4 -; CLEANUP-NEXT: [[TMP3:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP3]]) +; CLEANUP-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP1]] to ptr addrspace(21) ; CLEANUP-NEXT: [[TMP4:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CLEANUP-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4 -; CLEANUP-NEXT: store i32 [[TMP5]], ptr @CONTINUATION_STATE, align 4 -; CLEANUP-NEXT: [[TMP6:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CLEANUP-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4 -; CLEANUP-NEXT: store i32 [[TMP7]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CLEANUP-NEXT: [[TMP8:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 -; CLEANUP-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4 -; CLEANUP-NEXT: store i32 [[TMP9]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 2), align 4 -; CLEANUP-NEXT: [[TMP10:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 -; CLEANUP-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4 -; CLEANUP-NEXT: store i32 [[TMP11]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 3), align 4 -; CLEANUP-NEXT: [[TMP12:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 -; CLEANUP-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4 -; CLEANUP-NEXT: store i32 [[TMP13]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 4), align 4 -; CLEANUP-NEXT: [[TMP14:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 -; CLEANUP-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4 -; CLEANUP-NEXT: store i32 [[TMP15]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 5), align 4 -; CLEANUP-NEXT: [[TMP16:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4 -; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP17]], i64 ptrtoint (ptr @simple_await.resume.0 to i64)), !continuation.registercount !3, !continuation.returnedRegistercount !3 +; CLEANUP-NEXT: [[TMP5:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 0 +; CLEANUP-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4 +; CLEANUP-NEXT: store i32 [[TMP6]], ptr addrspace(21) [[TMP5]], align 4 +; CLEANUP-NEXT: [[TMP7:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CLEANUP-NEXT: [[TMP8:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 1 +; CLEANUP-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4 +; CLEANUP-NEXT: store i32 [[TMP9]], ptr addrspace(21) [[TMP8]], align 4 +; CLEANUP-NEXT: [[TMP10:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; CLEANUP-NEXT: [[TMP11:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 2 +; CLEANUP-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP10]], align 4 +; CLEANUP-NEXT: store i32 [[TMP12]], ptr addrspace(21) [[TMP11]], align 4 +; CLEANUP-NEXT: [[TMP13:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 +; CLEANUP-NEXT: [[TMP14:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 3 +; CLEANUP-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP13]], align 4 +; CLEANUP-NEXT: store i32 [[TMP15]], ptr addrspace(21) [[TMP14]], align 4 +; CLEANUP-NEXT: [[TMP16:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 +; CLEANUP-NEXT: [[TMP17:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 4 +; CLEANUP-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP16]], align 4 +; CLEANUP-NEXT: store i32 [[TMP18]], ptr addrspace(21) [[TMP17]], align 4 +; CLEANUP-NEXT: [[TMP19:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 +; CLEANUP-NEXT: [[TMP20:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 5 +; CLEANUP-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP19]], align 4 +; CLEANUP-NEXT: store i32 [[TMP21]], ptr addrspace(21) [[TMP20]], align 4 +; CLEANUP-NEXT: [[TMP22:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4 +; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP23]], i64 ptrtoint (ptr @simple_await.resume.0 to i64)), !continuation.registercount !2, !continuation.returnedRegistercount !2 ; CLEANUP-NEXT: unreachable ; ; -; CLEANUP-LABEL: define void @simple_await.resume.0( -; CLEANUP-SAME: i32 [[TMP0:%.*]]) !continuation.registercount !3 !continuation !4 { +; CLEANUP-LABEL: define dso_local void @simple_await.resume.0( +; CLEANUP-SAME: i32 [[TMP0:%.*]]) !continuation.registercount !2 !continuation !3 { ; CLEANUP-NEXT: entryresume.0: ; CLEANUP-NEXT: [[CONT_STATE:%.*]] = alloca [6 x i32], align 4 -; CLEANUP-NEXT: [[TMP1:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CLEANUP-NEXT: [[TMP2:%.*]] = load i32, ptr @CONTINUATION_STATE, align 4 -; CLEANUP-NEXT: store i32 [[TMP2]], ptr [[TMP1]], align 4 -; CLEANUP-NEXT: [[TMP3:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CLEANUP-NEXT: [[TMP4:%.*]] = load i32, ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CLEANUP-NEXT: store i32 [[TMP4]], ptr [[TMP3]], align 4 -; CLEANUP-NEXT: [[TMP5:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 -; CLEANUP-NEXT: [[TMP6:%.*]] = load i32, ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 2), align 4 -; CLEANUP-NEXT: store i32 [[TMP6]], ptr [[TMP5]], align 4 -; CLEANUP-NEXT: [[TMP7:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 -; CLEANUP-NEXT: [[TMP8:%.*]] = load i32, ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 3), align 4 -; CLEANUP-NEXT: store i32 [[TMP8]], ptr [[TMP7]], align 4 -; CLEANUP-NEXT: [[TMP9:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 -; CLEANUP-NEXT: [[TMP10:%.*]] = load i32, ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 4), align 4 +; CLEANUP-NEXT: [[TMP1:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4 +; CLEANUP-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(21) +; CLEANUP-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP3]], i64 -24 +; CLEANUP-NEXT: [[TMP5:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 0 +; CLEANUP-NEXT: [[TMP6:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; CLEANUP-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(21) [[TMP5]], align 4 +; CLEANUP-NEXT: store i32 [[TMP7]], ptr [[TMP6]], align 4 +; CLEANUP-NEXT: [[TMP8:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 1 +; CLEANUP-NEXT: [[TMP9:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CLEANUP-NEXT: [[TMP10:%.*]] = load i32, ptr addrspace(21) [[TMP8]], align 4 ; CLEANUP-NEXT: store i32 [[TMP10]], ptr [[TMP9]], align 4 -; CLEANUP-NEXT: [[TMP11:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 -; CLEANUP-NEXT: [[TMP12:%.*]] = load i32, ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 5), align 4 -; CLEANUP-NEXT: store i32 [[TMP12]], ptr [[TMP11]], align 4 -; CLEANUP-NEXT: [[TMP13:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP13]]) -; CLEANUP-NEXT: [[TMP14:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4 -; CLEANUP-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], -24 -; CLEANUP-NEXT: store i32 [[TMP16]], ptr [[TMP14]], align 4 +; CLEANUP-NEXT: [[TMP11:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 2 +; CLEANUP-NEXT: [[TMP12:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; CLEANUP-NEXT: [[TMP13:%.*]] = load i32, ptr addrspace(21) [[TMP11]], align 4 +; CLEANUP-NEXT: store i32 [[TMP13]], ptr [[TMP12]], align 4 +; CLEANUP-NEXT: [[TMP14:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 3 +; CLEANUP-NEXT: [[TMP15:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 +; CLEANUP-NEXT: [[TMP16:%.*]] = load i32, ptr addrspace(21) [[TMP14]], align 4 +; CLEANUP-NEXT: store i32 [[TMP16]], ptr [[TMP15]], align 4 +; CLEANUP-NEXT: [[TMP17:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 4 +; CLEANUP-NEXT: [[TMP18:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 +; CLEANUP-NEXT: [[TMP19:%.*]] = load i32, ptr addrspace(21) [[TMP17]], align 4 +; CLEANUP-NEXT: store i32 [[TMP19]], ptr [[TMP18]], align 4 +; CLEANUP-NEXT: [[TMP20:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 5 +; CLEANUP-NEXT: [[TMP21:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 +; CLEANUP-NEXT: [[TMP22:%.*]] = load i32, ptr addrspace(21) [[TMP20]], align 4 +; CLEANUP-NEXT: store i32 [[TMP22]], ptr [[TMP21]], align 4 +; CLEANUP-NEXT: [[TMP23:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4 +; CLEANUP-NEXT: [[TMP25:%.*]] = add i32 [[TMP24]], -24 +; CLEANUP-NEXT: store i32 [[TMP25]], ptr [[TMP23]], align 4 ; CLEANUP-NEXT: [[ARG_RELOAD_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; CLEANUP-NEXT: [[ARG_RELOAD:%.*]] = load <4 x i32>, ptr [[ARG_RELOAD_ADDR]], align 4 ; CLEANUP-NEXT: [[RETURNADDR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_FRAME]], ptr [[CONT_STATE]], i32 0, i32 1 ; CLEANUP-NEXT: [[RETURNADDR_RELOAD:%.*]] = load i64, ptr [[RETURNADDR_RELOAD_ADDR]], align 4 -; CLEANUP-NEXT: call void @continuation.restore.continuation_state() -; CLEANUP-NEXT: [[TMP17:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4 -; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP18]], <4 x i32> [[ARG_RELOAD]]), !continuation.registercount !3 +; CLEANUP-NEXT: [[TMP26:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP26]], align 4 +; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP27]], <4 x i32> [[ARG_RELOAD]]), !continuation.registercount !2 ; CLEANUP-NEXT: unreachable ; ; ; CLEANUP-LABEL: define void @simple_await_entry( -; CLEANUP-SAME: <4 x i32> [[ARG:%.*]], ptr addrspace(1) [[MEM:%.*]]) !continuation.registercount !3 !continuation.entry !6 !continuation !7 !continuation.state !5 !continuation.stacksize !5 { +; CLEANUP-SAME: <4 x i32> [[ARG:%.*]], ptr addrspace(1) [[MEM:%.*]]) !continuation.registercount !2 !continuation.entry !5 !continuation !6 !continuation.state !4 !continuation.stacksize !4 { ; CLEANUP-NEXT: AllocaSpillBB: ; CLEANUP-NEXT: [[CONT_STATE:%.*]] = alloca [6 x i32], align 4 ; CLEANUP-NEXT: [[MEM_SPILL_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_ENTRY_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 1 @@ -125,60 +136,73 @@ define void @simple_await_entry(<4 x i32> %arg, <4 x i32> addrspace(1)* %mem) !c ; CLEANUP-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4 ; CLEANUP-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 24 ; CLEANUP-NEXT: store i32 [[TMP2]], ptr [[TMP0]], align 4 -; CLEANUP-NEXT: [[TMP3:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP3]]) +; CLEANUP-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP1]] to ptr addrspace(21) ; CLEANUP-NEXT: [[TMP4:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CLEANUP-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4 -; CLEANUP-NEXT: store i32 [[TMP5]], ptr @CONTINUATION_STATE, align 4 -; CLEANUP-NEXT: [[TMP6:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CLEANUP-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4 -; CLEANUP-NEXT: store i32 [[TMP7]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CLEANUP-NEXT: [[TMP8:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 -; CLEANUP-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4 -; CLEANUP-NEXT: store i32 [[TMP9]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 2), align 4 -; CLEANUP-NEXT: [[TMP10:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 -; CLEANUP-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4 -; CLEANUP-NEXT: store i32 [[TMP11]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 3), align 4 -; CLEANUP-NEXT: [[TMP12:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 -; CLEANUP-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4 -; CLEANUP-NEXT: store i32 [[TMP13]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 4), align 4 -; CLEANUP-NEXT: [[TMP14:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 -; CLEANUP-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4 -; CLEANUP-NEXT: store i32 [[TMP15]], ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 5), align 4 -; CLEANUP-NEXT: [[TMP16:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4 -; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP17]], i64 ptrtoint (ptr @simple_await_entry.resume.0 to i64)), !continuation.registercount !3, !continuation.returnedRegistercount !3 +; CLEANUP-NEXT: [[TMP5:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 0 +; CLEANUP-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4 +; CLEANUP-NEXT: store i32 [[TMP6]], ptr addrspace(21) [[TMP5]], align 4 +; CLEANUP-NEXT: [[TMP7:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CLEANUP-NEXT: [[TMP8:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 1 +; CLEANUP-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4 +; CLEANUP-NEXT: store i32 [[TMP9]], ptr addrspace(21) [[TMP8]], align 4 +; CLEANUP-NEXT: [[TMP10:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; CLEANUP-NEXT: [[TMP11:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 2 +; CLEANUP-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP10]], align 4 +; CLEANUP-NEXT: store i32 [[TMP12]], ptr addrspace(21) [[TMP11]], align 4 +; CLEANUP-NEXT: [[TMP13:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 +; CLEANUP-NEXT: [[TMP14:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 3 +; CLEANUP-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP13]], align 4 +; CLEANUP-NEXT: store i32 [[TMP15]], ptr addrspace(21) [[TMP14]], align 4 +; CLEANUP-NEXT: [[TMP16:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 +; CLEANUP-NEXT: [[TMP17:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 4 +; CLEANUP-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP16]], align 4 +; CLEANUP-NEXT: store i32 [[TMP18]], ptr addrspace(21) [[TMP17]], align 4 +; CLEANUP-NEXT: [[TMP19:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 +; CLEANUP-NEXT: [[TMP20:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 5 +; CLEANUP-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP19]], align 4 +; CLEANUP-NEXT: store i32 [[TMP21]], ptr addrspace(21) [[TMP20]], align 4 +; CLEANUP-NEXT: [[TMP22:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4 +; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP23]], i64 ptrtoint (ptr @simple_await_entry.resume.0 to i64)), !continuation.registercount !2, !continuation.returnedRegistercount !2 ; CLEANUP-NEXT: unreachable ; ; -; CLEANUP-LABEL: define void @simple_await_entry.resume.0( -; CLEANUP-SAME: i32 [[TMP0:%.*]]) !continuation.registercount !3 !continuation !7 { +; CLEANUP-LABEL: define dso_local void @simple_await_entry.resume.0( +; CLEANUP-SAME: i32 [[TMP0:%.*]]) !continuation.registercount !2 !continuation !6 { ; CLEANUP-NEXT: entryresume.0: ; CLEANUP-NEXT: [[CONT_STATE:%.*]] = alloca [6 x i32], align 4 -; CLEANUP-NEXT: [[TMP1:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CLEANUP-NEXT: [[TMP2:%.*]] = load i32, ptr @CONTINUATION_STATE, align 4 -; CLEANUP-NEXT: store i32 [[TMP2]], ptr [[TMP1]], align 4 -; CLEANUP-NEXT: [[TMP3:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CLEANUP-NEXT: [[TMP4:%.*]] = load i32, ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CLEANUP-NEXT: store i32 [[TMP4]], ptr [[TMP3]], align 4 -; CLEANUP-NEXT: [[TMP5:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 -; CLEANUP-NEXT: [[TMP6:%.*]] = load i32, ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 2), align 4 -; CLEANUP-NEXT: store i32 [[TMP6]], ptr [[TMP5]], align 4 -; CLEANUP-NEXT: [[TMP7:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 -; CLEANUP-NEXT: [[TMP8:%.*]] = load i32, ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 3), align 4 -; CLEANUP-NEXT: store i32 [[TMP8]], ptr [[TMP7]], align 4 -; CLEANUP-NEXT: [[TMP9:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 -; CLEANUP-NEXT: [[TMP10:%.*]] = load i32, ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 4), align 4 +; CLEANUP-NEXT: [[TMP1:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4 +; CLEANUP-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(21) +; CLEANUP-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP3]], i64 -24 +; CLEANUP-NEXT: [[TMP5:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 0 +; CLEANUP-NEXT: [[TMP6:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; CLEANUP-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(21) [[TMP5]], align 4 +; CLEANUP-NEXT: store i32 [[TMP7]], ptr [[TMP6]], align 4 +; CLEANUP-NEXT: [[TMP8:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 1 +; CLEANUP-NEXT: [[TMP9:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CLEANUP-NEXT: [[TMP10:%.*]] = load i32, ptr addrspace(21) [[TMP8]], align 4 ; CLEANUP-NEXT: store i32 [[TMP10]], ptr [[TMP9]], align 4 -; CLEANUP-NEXT: [[TMP11:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 -; CLEANUP-NEXT: [[TMP12:%.*]] = load i32, ptr getelementptr inbounds ([6 x i32], ptr @CONTINUATION_STATE, i32 0, i32 5), align 4 -; CLEANUP-NEXT: store i32 [[TMP12]], ptr [[TMP11]], align 4 -; CLEANUP-NEXT: [[TMP13:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP13]]) -; CLEANUP-NEXT: [[TMP14:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4 -; CLEANUP-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], -24 -; CLEANUP-NEXT: store i32 [[TMP16]], ptr [[TMP14]], align 4 +; CLEANUP-NEXT: [[TMP11:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 2 +; CLEANUP-NEXT: [[TMP12:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; CLEANUP-NEXT: [[TMP13:%.*]] = load i32, ptr addrspace(21) [[TMP11]], align 4 +; CLEANUP-NEXT: store i32 [[TMP13]], ptr [[TMP12]], align 4 +; CLEANUP-NEXT: [[TMP14:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 3 +; CLEANUP-NEXT: [[TMP15:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 +; CLEANUP-NEXT: [[TMP16:%.*]] = load i32, ptr addrspace(21) [[TMP14]], align 4 +; CLEANUP-NEXT: store i32 [[TMP16]], ptr [[TMP15]], align 4 +; CLEANUP-NEXT: [[TMP17:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 4 +; CLEANUP-NEXT: [[TMP18:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 +; CLEANUP-NEXT: [[TMP19:%.*]] = load i32, ptr addrspace(21) [[TMP17]], align 4 +; CLEANUP-NEXT: store i32 [[TMP19]], ptr [[TMP18]], align 4 +; CLEANUP-NEXT: [[TMP20:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 5 +; CLEANUP-NEXT: [[TMP21:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 +; CLEANUP-NEXT: [[TMP22:%.*]] = load i32, ptr addrspace(21) [[TMP20]], align 4 +; CLEANUP-NEXT: store i32 [[TMP22]], ptr [[TMP21]], align 4 +; CLEANUP-NEXT: [[TMP23:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4 +; CLEANUP-NEXT: [[TMP25:%.*]] = add i32 [[TMP24]], -24 +; CLEANUP-NEXT: store i32 [[TMP25]], ptr [[TMP23]], align 4 ; CLEANUP-NEXT: [[MEM_RELOAD_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_ENTRY_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 1 ; CLEANUP-NEXT: [[MEM_RELOAD:%.*]] = load ptr addrspace(1), ptr [[MEM_RELOAD_ADDR]], align 4 ; CLEANUP-NEXT: [[ARG_RELOAD_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_ENTRY_FRAME]], ptr [[CONT_STATE]], i32 0, i32 0 @@ -194,137 +218,87 @@ define void @simple_await_entry(<4 x i32> %arg, <4 x i32> addrspace(1)* %mem) !c ; POST-PROCESS-NEXT: [[CONT_STATE:%.*]] = alloca [6 x i32], align 4 ; POST-PROCESS-NEXT: [[CSP:%.*]] = alloca i32, align 4 ; POST-PROCESS-NEXT: store i32 [[CSPINIT]], ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP0:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], 0 -; POST-PROCESS-NEXT: store i32 [[TMP1]], ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP2:%.*]] = inttoptr i32 [[TMP0]] to ptr addrspace(21) ; POST-PROCESS-NEXT: [[ARG_SPILL_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; POST-PROCESS-NEXT: store <4 x i32> [[ARG]], ptr [[ARG_SPILL_ADDR]], align 4 ; POST-PROCESS-NEXT: [[RETURNADDR_SPILL_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_FRAME]], ptr [[CONT_STATE]], i32 0, i32 1 ; POST-PROCESS-NEXT: store i64 [[RETURNADDR]], ptr [[RETURNADDR_SPILL_ADDR]], align 4 -; POST-PROCESS-NEXT: [[TMP3:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 24 -; POST-PROCESS-NEXT: store i32 [[TMP4]], ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; POST-PROCESS-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4 -; POST-PROCESS-NEXT: [[TMP7:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP8:%.*]] = inttoptr i32 [[TMP7]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP8]], i64 -24 -; POST-PROCESS-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP9]], i32 0 -; POST-PROCESS-NEXT: store i32 [[TMP6]], ptr addrspace(21) [[TMP10]], align 4 -; POST-PROCESS-NEXT: [[TMP11:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; POST-PROCESS-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4 -; POST-PROCESS-NEXT: [[TMP13:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP14:%.*]] = inttoptr i32 [[TMP13]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP15:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP14]], i64 -24 -; POST-PROCESS-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP15]], i32 0 -; POST-PROCESS-NEXT: [[TMP17:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP16]], i32 0, i32 1 -; POST-PROCESS-NEXT: store i32 [[TMP12]], ptr addrspace(21) [[TMP17]], align 4 -; POST-PROCESS-NEXT: [[TMP18:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 -; POST-PROCESS-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP18]], align 4 -; POST-PROCESS-NEXT: [[TMP20:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP21:%.*]] = inttoptr i32 [[TMP20]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP21]], i64 -24 -; POST-PROCESS-NEXT: [[TMP23:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP22]], i32 0 -; POST-PROCESS-NEXT: [[TMP24:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP23]], i32 0, i32 2 -; POST-PROCESS-NEXT: store i32 [[TMP19]], ptr addrspace(21) [[TMP24]], align 4 -; POST-PROCESS-NEXT: [[TMP25:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 -; POST-PROCESS-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP25]], align 4 -; POST-PROCESS-NEXT: [[TMP27:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP28:%.*]] = inttoptr i32 [[TMP27]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP29:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP28]], i64 -24 -; POST-PROCESS-NEXT: [[TMP30:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP29]], i32 0 -; POST-PROCESS-NEXT: [[TMP31:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP30]], i32 0, i32 3 -; POST-PROCESS-NEXT: store i32 [[TMP26]], ptr addrspace(21) [[TMP31]], align 4 -; POST-PROCESS-NEXT: [[TMP32:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 -; POST-PROCESS-NEXT: [[TMP33:%.*]] = load i32, ptr [[TMP32]], align 4 -; POST-PROCESS-NEXT: [[TMP34:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP35:%.*]] = inttoptr i32 [[TMP34]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP36:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP35]], i64 -24 -; POST-PROCESS-NEXT: [[TMP37:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP36]], i32 0 -; POST-PROCESS-NEXT: [[TMP38:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP37]], i32 0, i32 4 -; POST-PROCESS-NEXT: store i32 [[TMP33]], ptr addrspace(21) [[TMP38]], align 4 -; POST-PROCESS-NEXT: [[TMP39:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 -; POST-PROCESS-NEXT: [[TMP40:%.*]] = load i32, ptr [[TMP39]], align 4 -; POST-PROCESS-NEXT: [[TMP41:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP42:%.*]] = inttoptr i32 [[TMP41]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP43:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP42]], i64 -24 -; POST-PROCESS-NEXT: [[TMP44:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP43]], i32 0 -; POST-PROCESS-NEXT: [[TMP45:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP44]], i32 0, i32 5 -; POST-PROCESS-NEXT: store i32 [[TMP40]], ptr addrspace(21) [[TMP45]], align 4 -; POST-PROCESS-NEXT: [[TMP46:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP47:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @async_fun to i64)) -; POST-PROCESS-NEXT: [[TMP48:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @simple_await.resume.0 to i64)) -; POST-PROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[TMP47]], i32 [[TMP46]], i64 [[TMP48]]), !continuation.registercount !2, !continuation.returnedRegistercount !2 +; POST-PROCESS-NEXT: [[TMP0:%.*]] = load i32, ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], 24 +; POST-PROCESS-NEXT: store i32 [[TMP1]], ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP2:%.*]] = inttoptr i32 [[TMP0]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP3:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; POST-PROCESS-NEXT: [[TMP4:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP2]], i32 0, i32 0 +; POST-PROCESS-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP3]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP5]], ptr addrspace(21) [[TMP4]], align 4 +; POST-PROCESS-NEXT: [[TMP6:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; POST-PROCESS-NEXT: [[TMP7:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP2]], i32 0, i32 1 +; POST-PROCESS-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP6]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP8]], ptr addrspace(21) [[TMP7]], align 4 +; POST-PROCESS-NEXT: [[TMP9:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; POST-PROCESS-NEXT: [[TMP10:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP2]], i32 0, i32 2 +; POST-PROCESS-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP9]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP11]], ptr addrspace(21) [[TMP10]], align 4 +; POST-PROCESS-NEXT: [[TMP12:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 +; POST-PROCESS-NEXT: [[TMP13:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP2]], i32 0, i32 3 +; POST-PROCESS-NEXT: [[TMP14:%.*]] = load i32, ptr [[TMP12]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP14]], ptr addrspace(21) [[TMP13]], align 4 +; POST-PROCESS-NEXT: [[TMP15:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 +; POST-PROCESS-NEXT: [[TMP16:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP2]], i32 0, i32 4 +; POST-PROCESS-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP15]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP17]], ptr addrspace(21) [[TMP16]], align 4 +; POST-PROCESS-NEXT: [[TMP18:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 +; POST-PROCESS-NEXT: [[TMP19:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP2]], i32 0, i32 5 +; POST-PROCESS-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP18]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP20]], ptr addrspace(21) [[TMP19]], align 4 +; POST-PROCESS-NEXT: [[TMP21:%.*]] = load i32, ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP22:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @async_fun to i64)) +; POST-PROCESS-NEXT: [[TMP23:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @simple_await.resume.0 to i64)) +; POST-PROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[TMP22]], i32 [[TMP21]], i64 [[TMP23]]), !continuation.registercount !2, !continuation.returnedRegistercount !2 ; POST-PROCESS-NEXT: unreachable ; ; -; POST-PROCESS-LABEL: define void @simple_await.resume.0( +; POST-PROCESS-LABEL: define dso_local void @simple_await.resume.0( ; POST-PROCESS-SAME: i32 [[TMP0:%.*]]) !continuation.registercount !2 !continuation !3 { ; POST-PROCESS-NEXT: entryresume.0: ; POST-PROCESS-NEXT: [[CONT_STATE:%.*]] = alloca [6 x i32], align 4 ; POST-PROCESS-NEXT: [[CSP:%.*]] = alloca i32, align 4 ; POST-PROCESS-NEXT: store i32 [[TMP0]], ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP1:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; POST-PROCESS-NEXT: [[TMP2:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP3]], i64 -24 -; POST-PROCESS-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP4]], i32 0 -; POST-PROCESS-NEXT: [[TMP6:%.*]] = load i32, ptr addrspace(21) [[TMP5]], align 4 -; POST-PROCESS-NEXT: store i32 [[TMP6]], ptr [[TMP1]], align 4 -; POST-PROCESS-NEXT: [[TMP7:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; POST-PROCESS-NEXT: [[TMP8:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP9:%.*]] = inttoptr i32 [[TMP8]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP9]], i64 -24 -; POST-PROCESS-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP10]], i32 0 -; POST-PROCESS-NEXT: [[TMP12:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP11]], i32 0, i32 1 -; POST-PROCESS-NEXT: [[TMP13:%.*]] = load i32, ptr addrspace(21) [[TMP12]], align 4 -; POST-PROCESS-NEXT: store i32 [[TMP13]], ptr [[TMP7]], align 4 -; POST-PROCESS-NEXT: [[TMP14:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 -; POST-PROCESS-NEXT: [[TMP15:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP16:%.*]] = inttoptr i32 [[TMP15]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP16]], i64 -24 -; POST-PROCESS-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP17]], i32 0 -; POST-PROCESS-NEXT: [[TMP19:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP18]], i32 0, i32 2 -; POST-PROCESS-NEXT: [[TMP20:%.*]] = load i32, ptr addrspace(21) [[TMP19]], align 4 -; POST-PROCESS-NEXT: store i32 [[TMP20]], ptr [[TMP14]], align 4 -; POST-PROCESS-NEXT: [[TMP21:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 +; POST-PROCESS-NEXT: [[TMP1:%.*]] = load i32, ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP2:%.*]] = inttoptr i32 [[TMP1]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP2]], i64 -24 +; POST-PROCESS-NEXT: [[TMP4:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 0 +; POST-PROCESS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; POST-PROCESS-NEXT: [[TMP6:%.*]] = load i32, ptr addrspace(21) [[TMP4]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP6]], ptr [[TMP5]], align 4 +; POST-PROCESS-NEXT: [[TMP7:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 1 +; POST-PROCESS-NEXT: [[TMP8:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; POST-PROCESS-NEXT: [[TMP9:%.*]] = load i32, ptr addrspace(21) [[TMP7]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP9]], ptr [[TMP8]], align 4 +; POST-PROCESS-NEXT: [[TMP10:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 2 +; POST-PROCESS-NEXT: [[TMP11:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; POST-PROCESS-NEXT: [[TMP12:%.*]] = load i32, ptr addrspace(21) [[TMP10]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP12]], ptr [[TMP11]], align 4 +; POST-PROCESS-NEXT: [[TMP13:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 3 +; POST-PROCESS-NEXT: [[TMP14:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 +; POST-PROCESS-NEXT: [[TMP15:%.*]] = load i32, ptr addrspace(21) [[TMP13]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP15]], ptr [[TMP14]], align 4 +; POST-PROCESS-NEXT: [[TMP16:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 4 +; POST-PROCESS-NEXT: [[TMP17:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 +; POST-PROCESS-NEXT: [[TMP18:%.*]] = load i32, ptr addrspace(21) [[TMP16]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP18]], ptr [[TMP17]], align 4 +; POST-PROCESS-NEXT: [[TMP19:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 5 +; POST-PROCESS-NEXT: [[TMP20:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 +; POST-PROCESS-NEXT: [[TMP21:%.*]] = load i32, ptr addrspace(21) [[TMP19]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP21]], ptr [[TMP20]], align 4 ; POST-PROCESS-NEXT: [[TMP22:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP23:%.*]] = inttoptr i32 [[TMP22]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP23]], i64 -24 -; POST-PROCESS-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP24]], i32 0 -; POST-PROCESS-NEXT: [[TMP26:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP25]], i32 0, i32 3 -; POST-PROCESS-NEXT: [[TMP27:%.*]] = load i32, ptr addrspace(21) [[TMP26]], align 4 -; POST-PROCESS-NEXT: store i32 [[TMP27]], ptr [[TMP21]], align 4 -; POST-PROCESS-NEXT: [[TMP28:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 -; POST-PROCESS-NEXT: [[TMP29:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP30:%.*]] = inttoptr i32 [[TMP29]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP31:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP30]], i64 -24 -; POST-PROCESS-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP31]], i32 0 -; POST-PROCESS-NEXT: [[TMP33:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP32]], i32 0, i32 4 -; POST-PROCESS-NEXT: [[TMP34:%.*]] = load i32, ptr addrspace(21) [[TMP33]], align 4 -; POST-PROCESS-NEXT: store i32 [[TMP34]], ptr [[TMP28]], align 4 -; POST-PROCESS-NEXT: [[TMP35:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 -; POST-PROCESS-NEXT: [[TMP36:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP37:%.*]] = inttoptr i32 [[TMP36]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP38:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP37]], i64 -24 -; POST-PROCESS-NEXT: [[TMP39:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP38]], i32 0 -; POST-PROCESS-NEXT: [[TMP40:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP39]], i32 0, i32 5 -; POST-PROCESS-NEXT: [[TMP41:%.*]] = load i32, ptr addrspace(21) [[TMP40]], align 4 -; POST-PROCESS-NEXT: store i32 [[TMP41]], ptr [[TMP35]], align 4 -; POST-PROCESS-NEXT: [[TMP42:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP43:%.*]] = add i32 [[TMP42]], -24 -; POST-PROCESS-NEXT: store i32 [[TMP43]], ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP23:%.*]] = add i32 [[TMP22]], -24 +; POST-PROCESS-NEXT: store i32 [[TMP23]], ptr [[CSP]], align 4 ; POST-PROCESS-NEXT: [[ARG_RELOAD_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; POST-PROCESS-NEXT: [[ARG_RELOAD:%.*]] = load <4 x i32>, ptr [[ARG_RELOAD_ADDR]], align 4 ; POST-PROCESS-NEXT: [[RETURNADDR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_FRAME]], ptr [[CONT_STATE]], i32 0, i32 1 ; POST-PROCESS-NEXT: [[RETURNADDR_RELOAD:%.*]] = load i64, ptr [[RETURNADDR_RELOAD_ADDR]], align 4 -; POST-PROCESS-NEXT: [[TMP44:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP45:%.*]] = add i32 [[TMP44]], 0 -; POST-PROCESS-NEXT: store i32 [[TMP45]], ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP46:%.*]] = inttoptr i32 [[TMP45]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP47:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP47]], <4 x i32> [[ARG_RELOAD]]), !continuation.registercount !2 +; POST-PROCESS-NEXT: [[TMP24:%.*]] = load i32, ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP24]], <4 x i32> [[ARG_RELOAD]]), !continuation.registercount !2 ; POST-PROCESS-NEXT: unreachable ; ; @@ -342,116 +316,74 @@ define void @simple_await_entry(<4 x i32> %arg, <4 x i32> addrspace(1)* %mem) !c ; POST-PROCESS-NEXT: [[TMP1:%.*]] = load i32, ptr [[CSP]], align 4 ; POST-PROCESS-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 24 ; POST-PROCESS-NEXT: store i32 [[TMP2]], ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP3:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; POST-PROCESS-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4 -; POST-PROCESS-NEXT: [[TMP5:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP6:%.*]] = inttoptr i32 [[TMP5]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP6]], i64 -24 -; POST-PROCESS-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP7]], i32 0 -; POST-PROCESS-NEXT: store i32 [[TMP4]], ptr addrspace(21) [[TMP8]], align 4 -; POST-PROCESS-NEXT: [[TMP9:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; POST-PROCESS-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4 -; POST-PROCESS-NEXT: [[TMP11:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP12:%.*]] = inttoptr i32 [[TMP11]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP12]], i64 -24 -; POST-PROCESS-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP13]], i32 0 -; POST-PROCESS-NEXT: [[TMP15:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP14]], i32 0, i32 1 -; POST-PROCESS-NEXT: store i32 [[TMP10]], ptr addrspace(21) [[TMP15]], align 4 -; POST-PROCESS-NEXT: [[TMP16:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 -; POST-PROCESS-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4 -; POST-PROCESS-NEXT: [[TMP18:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP19:%.*]] = inttoptr i32 [[TMP18]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP20:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP19]], i64 -24 -; POST-PROCESS-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP20]], i32 0 -; POST-PROCESS-NEXT: [[TMP22:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP21]], i32 0, i32 2 -; POST-PROCESS-NEXT: store i32 [[TMP17]], ptr addrspace(21) [[TMP22]], align 4 -; POST-PROCESS-NEXT: [[TMP23:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 -; POST-PROCESS-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4 -; POST-PROCESS-NEXT: [[TMP25:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP26:%.*]] = inttoptr i32 [[TMP25]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP26]], i64 -24 -; POST-PROCESS-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP27]], i32 0 -; POST-PROCESS-NEXT: [[TMP29:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP28]], i32 0, i32 3 -; POST-PROCESS-NEXT: store i32 [[TMP24]], ptr addrspace(21) [[TMP29]], align 4 -; POST-PROCESS-NEXT: [[TMP30:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 -; POST-PROCESS-NEXT: [[TMP31:%.*]] = load i32, ptr [[TMP30]], align 4 -; POST-PROCESS-NEXT: [[TMP32:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP33:%.*]] = inttoptr i32 [[TMP32]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP34:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP33]], i64 -24 -; POST-PROCESS-NEXT: [[TMP35:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP34]], i32 0 -; POST-PROCESS-NEXT: [[TMP36:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP35]], i32 0, i32 4 -; POST-PROCESS-NEXT: store i32 [[TMP31]], ptr addrspace(21) [[TMP36]], align 4 -; POST-PROCESS-NEXT: [[TMP37:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 -; POST-PROCESS-NEXT: [[TMP38:%.*]] = load i32, ptr [[TMP37]], align 4 -; POST-PROCESS-NEXT: [[TMP39:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP40:%.*]] = inttoptr i32 [[TMP39]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP41:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP40]], i64 -24 -; POST-PROCESS-NEXT: [[TMP42:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP41]], i32 0 -; POST-PROCESS-NEXT: [[TMP43:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP42]], i32 0, i32 5 -; POST-PROCESS-NEXT: store i32 [[TMP38]], ptr addrspace(21) [[TMP43]], align 4 -; POST-PROCESS-NEXT: [[TMP44:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP45:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @async_fun to i64)) -; POST-PROCESS-NEXT: [[TMP46:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @simple_await_entry.resume.0 to i64)) -; POST-PROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[TMP45]], i32 [[TMP44]], i64 [[TMP46]]), !continuation.registercount !2, !continuation.returnedRegistercount !2 +; POST-PROCESS-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP1]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP4:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; POST-PROCESS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 0 +; POST-PROCESS-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP6]], ptr addrspace(21) [[TMP5]], align 4 +; POST-PROCESS-NEXT: [[TMP7:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; POST-PROCESS-NEXT: [[TMP8:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 1 +; POST-PROCESS-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP9]], ptr addrspace(21) [[TMP8]], align 4 +; POST-PROCESS-NEXT: [[TMP10:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; POST-PROCESS-NEXT: [[TMP11:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 2 +; POST-PROCESS-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP10]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP12]], ptr addrspace(21) [[TMP11]], align 4 +; POST-PROCESS-NEXT: [[TMP13:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 +; POST-PROCESS-NEXT: [[TMP14:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 3 +; POST-PROCESS-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP13]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP15]], ptr addrspace(21) [[TMP14]], align 4 +; POST-PROCESS-NEXT: [[TMP16:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 +; POST-PROCESS-NEXT: [[TMP17:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 4 +; POST-PROCESS-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP16]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP18]], ptr addrspace(21) [[TMP17]], align 4 +; POST-PROCESS-NEXT: [[TMP19:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 +; POST-PROCESS-NEXT: [[TMP20:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 5 +; POST-PROCESS-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP19]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP21]], ptr addrspace(21) [[TMP20]], align 4 +; POST-PROCESS-NEXT: [[TMP22:%.*]] = load i32, ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP23:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @async_fun to i64)) +; POST-PROCESS-NEXT: [[TMP24:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @simple_await_entry.resume.0 to i64)) +; POST-PROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[TMP23]], i32 [[TMP22]], i64 [[TMP24]]), !continuation.registercount !2, !continuation.returnedRegistercount !2 ; POST-PROCESS-NEXT: unreachable ; ; -; POST-PROCESS-LABEL: define void @simple_await_entry.resume.0( +; POST-PROCESS-LABEL: define dso_local void @simple_await_entry.resume.0( ; POST-PROCESS-SAME: i32 [[TMP0:%.*]]) !continuation.registercount !2 !continuation !6 { ; POST-PROCESS-NEXT: entryresume.0: ; POST-PROCESS-NEXT: [[CONT_STATE:%.*]] = alloca [6 x i32], align 4 ; POST-PROCESS-NEXT: [[CSP:%.*]] = alloca i32, align 4 ; POST-PROCESS-NEXT: store i32 [[TMP0]], ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP1:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; POST-PROCESS-NEXT: [[TMP2:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP3]], i64 -24 -; POST-PROCESS-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP4]], i32 0 -; POST-PROCESS-NEXT: [[TMP6:%.*]] = load i32, ptr addrspace(21) [[TMP5]], align 4 -; POST-PROCESS-NEXT: store i32 [[TMP6]], ptr [[TMP1]], align 4 -; POST-PROCESS-NEXT: [[TMP7:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; POST-PROCESS-NEXT: [[TMP8:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP9:%.*]] = inttoptr i32 [[TMP8]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP9]], i64 -24 -; POST-PROCESS-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP10]], i32 0 -; POST-PROCESS-NEXT: [[TMP12:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP11]], i32 0, i32 1 -; POST-PROCESS-NEXT: [[TMP13:%.*]] = load i32, ptr addrspace(21) [[TMP12]], align 4 -; POST-PROCESS-NEXT: store i32 [[TMP13]], ptr [[TMP7]], align 4 -; POST-PROCESS-NEXT: [[TMP14:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 -; POST-PROCESS-NEXT: [[TMP15:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP16:%.*]] = inttoptr i32 [[TMP15]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP16]], i64 -24 -; POST-PROCESS-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP17]], i32 0 -; POST-PROCESS-NEXT: [[TMP19:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP18]], i32 0, i32 2 -; POST-PROCESS-NEXT: [[TMP20:%.*]] = load i32, ptr addrspace(21) [[TMP19]], align 4 -; POST-PROCESS-NEXT: store i32 [[TMP20]], ptr [[TMP14]], align 4 -; POST-PROCESS-NEXT: [[TMP21:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 +; POST-PROCESS-NEXT: [[TMP1:%.*]] = load i32, ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP2:%.*]] = inttoptr i32 [[TMP1]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP2]], i64 -24 +; POST-PROCESS-NEXT: [[TMP4:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 0 +; POST-PROCESS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; POST-PROCESS-NEXT: [[TMP6:%.*]] = load i32, ptr addrspace(21) [[TMP4]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP6]], ptr [[TMP5]], align 4 +; POST-PROCESS-NEXT: [[TMP7:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 1 +; POST-PROCESS-NEXT: [[TMP8:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; POST-PROCESS-NEXT: [[TMP9:%.*]] = load i32, ptr addrspace(21) [[TMP7]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP9]], ptr [[TMP8]], align 4 +; POST-PROCESS-NEXT: [[TMP10:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 2 +; POST-PROCESS-NEXT: [[TMP11:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; POST-PROCESS-NEXT: [[TMP12:%.*]] = load i32, ptr addrspace(21) [[TMP10]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP12]], ptr [[TMP11]], align 4 +; POST-PROCESS-NEXT: [[TMP13:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 3 +; POST-PROCESS-NEXT: [[TMP14:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 3 +; POST-PROCESS-NEXT: [[TMP15:%.*]] = load i32, ptr addrspace(21) [[TMP13]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP15]], ptr [[TMP14]], align 4 +; POST-PROCESS-NEXT: [[TMP16:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 4 +; POST-PROCESS-NEXT: [[TMP17:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 +; POST-PROCESS-NEXT: [[TMP18:%.*]] = load i32, ptr addrspace(21) [[TMP16]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP18]], ptr [[TMP17]], align 4 +; POST-PROCESS-NEXT: [[TMP19:%.*]] = getelementptr inbounds [6 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 5 +; POST-PROCESS-NEXT: [[TMP20:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 +; POST-PROCESS-NEXT: [[TMP21:%.*]] = load i32, ptr addrspace(21) [[TMP19]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP21]], ptr [[TMP20]], align 4 ; POST-PROCESS-NEXT: [[TMP22:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP23:%.*]] = inttoptr i32 [[TMP22]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP23]], i64 -24 -; POST-PROCESS-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP24]], i32 0 -; POST-PROCESS-NEXT: [[TMP26:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP25]], i32 0, i32 3 -; POST-PROCESS-NEXT: [[TMP27:%.*]] = load i32, ptr addrspace(21) [[TMP26]], align 4 -; POST-PROCESS-NEXT: store i32 [[TMP27]], ptr [[TMP21]], align 4 -; POST-PROCESS-NEXT: [[TMP28:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 4 -; POST-PROCESS-NEXT: [[TMP29:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP30:%.*]] = inttoptr i32 [[TMP29]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP31:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP30]], i64 -24 -; POST-PROCESS-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP31]], i32 0 -; POST-PROCESS-NEXT: [[TMP33:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP32]], i32 0, i32 4 -; POST-PROCESS-NEXT: [[TMP34:%.*]] = load i32, ptr addrspace(21) [[TMP33]], align 4 -; POST-PROCESS-NEXT: store i32 [[TMP34]], ptr [[TMP28]], align 4 -; POST-PROCESS-NEXT: [[TMP35:%.*]] = getelementptr inbounds [6 x i32], ptr [[CONT_STATE]], i32 0, i32 5 -; POST-PROCESS-NEXT: [[TMP36:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP37:%.*]] = inttoptr i32 [[TMP36]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP38:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP37]], i64 -24 -; POST-PROCESS-NEXT: [[TMP39:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP38]], i32 0 -; POST-PROCESS-NEXT: [[TMP40:%.*]] = getelementptr [6 x i32], ptr addrspace(21) [[TMP39]], i32 0, i32 5 -; POST-PROCESS-NEXT: [[TMP41:%.*]] = load i32, ptr addrspace(21) [[TMP40]], align 4 -; POST-PROCESS-NEXT: store i32 [[TMP41]], ptr [[TMP35]], align 4 -; POST-PROCESS-NEXT: [[TMP42:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP43:%.*]] = add i32 [[TMP42]], -24 -; POST-PROCESS-NEXT: store i32 [[TMP43]], ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP23:%.*]] = add i32 [[TMP22]], -24 +; POST-PROCESS-NEXT: store i32 [[TMP23]], ptr [[CSP]], align 4 ; POST-PROCESS-NEXT: [[MEM_RELOAD_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_ENTRY_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 1 ; POST-PROCESS-NEXT: [[MEM_RELOAD:%.*]] = load ptr addrspace(1), ptr [[MEM_RELOAD_ADDR]], align 4 ; POST-PROCESS-NEXT: [[ARG_RELOAD_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_ENTRY_FRAME]], ptr [[CONT_STATE]], i32 0, i32 0 diff --git a/shared/continuations/test/dx/continuation-without-await.ll b/shared/continuations/test/dx/continuation-without-await.ll index 13ed2b6bef..3d6ad894f7 100644 --- a/shared/continuations/test/dx/continuation-without-await.ll +++ b/shared/continuations/test/dx/continuation-without-await.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 3 -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2>%t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2> %t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s ; RUN: count 0 < %t0.stderr -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,remove-types-metadata' \ -; RUN: -S %s 2>%t1.stderr | FileCheck -check-prefix=CLEANUP %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,remove-types-metadata' \ +; RUN: -S %s 2> %t1.stderr | FileCheck -check-prefix=CLEANUP %s ; RUN: count 0 < %t1.stderr -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,remove-types-metadata' \ -; RUN: -S %s 2>%t2.stderr | FileCheck -check-prefix=SAVESTATE %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,remove-types-metadata' \ +; RUN: -S %s 2> %t2.stderr | FileCheck -check-prefix=SAVESTATE %s ; RUN: count 0 < %t2.stderr ; @called and @main_no_call must be marked as continuation and end with a continue call to the return address @@ -126,78 +126,45 @@ attributes #2 = { nounwind } ; LOWERRAYTRACINGPIPELINE-NEXT: ret i32 5 ; ; -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_TraceRay( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i64 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]], i32 [[TMP3:%.*]], i32 [[TMP4:%.*]], i32 [[TMP5:%.*]], float [[TMP6:%.*]], float [[TMP7:%.*]], float [[TMP8:%.*]], float [[TMP9:%.*]], float [[TMP10:%.*]], float [[TMP11:%.*]], float [[TMP12:%.*]], float [[TMP13:%.*]]) { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[NEWDATA:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @_AmdAwaitTraversal(i64 4, [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_CallShader.struct.TheirParams( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i32 [[TMP0:%.*]], ptr [[TMP1:%.*]]) { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_THEIRPARAMS:%.*]], ptr [[TMP1]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = load [1 x i32], ptr [[TMP3]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store [1 x i32] [[TMP4]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = call ptr inttoptr (i64 2 to ptr)([[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]]), !continuation.registercount !19, !continuation.returnedRegistercount !19 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @await.struct.DispatchSystemData(ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_THEIRPARAMS]] poison, ptr [[TMP1]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_THEIRPARAMS]], ptr [[TMP1]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = load [1 x i32], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store [1 x i32] [[TMP8]], ptr [[TMP7]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP6]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_CallShader( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i32 [[TMP0:%.*]]) { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[NEWDATA:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @_AmdAwaitShader(i64 2, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define void @main() !lgc.rt.shaderstage !9 !continuation.entry !20 !continuation.registercount !9 !continuation !21 { +; LOWERRAYTRACINGPIPELINE-LABEL: define void @main( +; LOWERRAYTRACINGPIPELINE-SAME: ) !lgc.rt.shaderstage [[META9:![0-9]+]] !continuation.entry [[META19:![0-9]+]] !continuation.registercount [[META9]] !continuation [[META20:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[PARAMS:%.*]] = alloca [[STRUCT_THEIRPARAMS:%.*]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP1:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_THEIRPARAMS]] zeroinitializer, ptr [[PARAMS]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_CallShader.struct.TheirParams(ptr [[SYSTEM_DATA_ALLOCA]], i32 1, ptr [[PARAMS]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define void @main_no_call() !lgc.rt.shaderstage !9 !continuation.entry !20 !continuation.registercount !9 !continuation !22 { +; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA_I:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_THEIRPARAMS]], ptr [[PARAMS]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = load [1 x i32], ptr [[TMP2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [1 x i32] [[TMP3]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = call ptr inttoptr (i64 2 to ptr)([[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I]]), !continuation.registercount [[META21:![0-9]+]], !continuation.returnedRegistercount !21 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @await.struct.DispatchSystemData(ptr [[TMP4]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_THEIRPARAMS]] poison, ptr [[PARAMS]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_THEIRPARAMS]], ptr [[PARAMS]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = load [1 x i32], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [1 x i32] [[TMP7]], ptr [[TMP6]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP5]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; LOWERRAYTRACINGPIPELINE-NEXT: ret void, !continuation.registercount [[META17:![0-9]+]] +; +; +; LOWERRAYTRACINGPIPELINE-LABEL: define void @main_no_call( +; LOWERRAYTRACINGPIPELINE-SAME: ) !lgc.rt.shaderstage [[META9]] !continuation.entry [[META19]] !continuation.registercount [[META9]] !continuation [[META22:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP1:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; LOWERRAYTRACINGPIPELINE-NEXT: ret void, !continuation.registercount [[META17]] ; ; ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.DispatchSystemData @called( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !lgc.rt.shaderstage !23 !continuation.registercount !24 !continuation !25 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !lgc.rt.shaderstage [[META23:![0-9]+]] !continuation.registercount [[META24:![0-9]+]] !continuation [[META25:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_MYPARAMS:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP3]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_MYPARAMS]], ptr [[TMP2]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[TMP4]], i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[TMP5]], i64 0 @@ -223,7 +190,7 @@ attributes #2 = { nounwind } ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP18]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP19]], ptr getelementptr (i32, ptr @PAYLOAD, i64 2), align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP20:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP20]], !continuation.registercount !24 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP20]], !continuation.registercount [[META24]] ; ; ; CLEANUP-LABEL: define i32 @_cont_GetLocalRootIndex( @@ -231,7 +198,8 @@ attributes #2 = { nounwind } ; CLEANUP-NEXT: ret i32 5 ; ; -; CLEANUP-LABEL: define void @main() !lgc.rt.shaderstage !10 !continuation.entry !20 !continuation.registercount !10 !continuation !21 !continuation.state !10 { +; CLEANUP-LABEL: define void @main( +; CLEANUP-SAME: ) !lgc.rt.shaderstage [[META9:![0-9]+]] !continuation.entry [[META19:![0-9]+]] !continuation.registercount [[META9]] !continuation [[META20:![0-9]+]] !continuation.state [[META9]] { ; CLEANUP-NEXT: AllocaSpillBB: ; CLEANUP-NEXT: [[CONT_STATE:%.*]] = alloca [0 x i32], align 4 ; CLEANUP-NEXT: [[TMP0:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA:%.*]] @continuations.getSystemData.s_struct.DispatchSystemDatas() @@ -242,23 +210,24 @@ attributes #2 = { nounwind } ; CLEANUP-NEXT: store [1 x i32] [[DOTFCA_0_INSERT]], ptr @PAYLOAD, align 4 ; CLEANUP-NEXT: [[TMP1:%.*]] = call ptr @continuation.getContinuationStackOffset() ; CLEANUP-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4 -; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 2, i32 [[TMP2]], i64 ptrtoint (ptr @main.resume.0 to i64), [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I_FCA_0_INSERT]]), !continuation.registercount !22, !continuation.returnedRegistercount !22 +; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 2, i32 [[TMP2]], i64 ptrtoint (ptr @main.resume.0 to i64), [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I_FCA_0_INSERT]]), !continuation.registercount [[META21:![0-9]+]], !continuation.returnedRegistercount !21 ; CLEANUP-NEXT: unreachable ; ; -; CLEANUP-LABEL: define void @main.resume.0( -; CLEANUP-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage !10 !continuation.registercount !22 !continuation !21 { +; CLEANUP-LABEL: define dso_local void @main.resume.0( +; CLEANUP-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage [[META9]] !continuation.registercount [[META21]] !continuation [[META20]] { ; CLEANUP-NEXT: entryresume.0: ; CLEANUP-NEXT: [[CONT_STATE:%.*]] = alloca [0 x i32], align 4 ; CLEANUP-NEXT: [[TMP2:%.*]] = load [1 x i32], ptr @PAYLOAD, align 4 -; CLEANUP-NEXT: [[DOTFCA_0_EXTRACT3:%.*]] = extractvalue [1 x i32] [[TMP2]], 0 +; CLEANUP-NEXT: [[DOTFCA_0_EXTRACT4:%.*]] = extractvalue [1 x i32] [[TMP2]], 0 ; CLEANUP-NEXT: [[DOTFCA_0_EXTRACT1:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], 0 ; CLEANUP-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; CLEANUP-NEXT: call void @continuation.complete() ; CLEANUP-NEXT: unreachable ; ; -; CLEANUP-LABEL: define void @main_no_call() !lgc.rt.shaderstage !10 !continuation.entry !20 !continuation.registercount !10 !continuation !23 !continuation.state !10 { +; CLEANUP-LABEL: define void @main_no_call( +; CLEANUP-SAME: ) !lgc.rt.shaderstage [[META9]] !continuation.entry [[META19]] !continuation.registercount [[META9]] !continuation [[META22:![0-9]+]] !continuation.state [[META9]] { ; CLEANUP-NEXT: AllocaSpillBB: ; CLEANUP-NEXT: [[CONT_STATE:%.*]] = alloca [0 x i32], align 4 ; CLEANUP-NEXT: [[TMP0:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA:%.*]] @continuations.getSystemData.s_struct.DispatchSystemDatas() @@ -269,7 +238,7 @@ attributes #2 = { nounwind } ; ; ; CLEANUP-LABEL: define void @called( -; CLEANUP-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !lgc.rt.shaderstage !24 !continuation.registercount !25 !continuation !26 !continuation.state !10 { +; CLEANUP-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !lgc.rt.shaderstage [[META23:![0-9]+]] !continuation.registercount [[META24:![0-9]+]] !continuation [[META25:![0-9]+]] !continuation.state [[META9]] { ; CLEANUP-NEXT: AllocaSpillBB: ; CLEANUP-NEXT: [[CONT_STATE:%.*]] = alloca [0 x i32], align 4 ; CLEANUP-NEXT: [[TMP1:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() @@ -286,7 +255,7 @@ attributes #2 = { nounwind } ; CLEANUP-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, i32 [[DOTFCA_0_EXTRACT]], 0 ; CLEANUP-NEXT: [[TMP5:%.*]] = call ptr @continuation.getContinuationStackOffset() ; CLEANUP-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4 -; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP6]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount !25 +; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP6]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount [[META24]] ; CLEANUP-NEXT: unreachable ; ; @@ -295,7 +264,8 @@ attributes #2 = { nounwind } ; SAVESTATE-NEXT: ret i32 5 ; ; -; SAVESTATE-LABEL: define void @main() !lgc.rt.shaderstage !8 !continuation.entry !18 !continuation.registercount !8 !continuation !19 !continuation.state !8 { +; SAVESTATE-LABEL: define void @main( +; SAVESTATE-SAME: ) !lgc.rt.shaderstage [[META8:![0-9]+]] !continuation.entry [[META18:![0-9]+]] !continuation.registercount [[META8]] !continuation [[META19:![0-9]+]] !continuation.state [[META8]] { ; SAVESTATE-NEXT: AllocaSpillBB: ; SAVESTATE-NEXT: [[CONT_STATE:%.*]] = alloca [0 x i32], align 4 ; SAVESTATE-NEXT: [[CSP:%.*]] = alloca i32, align 4 @@ -309,24 +279,25 @@ attributes #2 = { nounwind } ; SAVESTATE-NEXT: [[DOTFCA_0_INSERT_FCA_0_EXTRACT:%.*]] = extractvalue [1 x i32] [[DOTFCA_0_INSERT]], 0 ; SAVESTATE-NEXT: store i32 [[DOTFCA_0_INSERT_FCA_0_EXTRACT]], ptr addrspace(20) @PAYLOAD, align 4 ; SAVESTATE-NEXT: [[TMP2:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: call void (i64, ...) @continuation.continue(i64 2, i32 [[TMP2]], i64 ptrtoint (ptr @main.resume.0 to i64), [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I_FCA_0_INSERT]]), !continuation.registercount !20, !continuation.returnedRegistercount !20 +; SAVESTATE-NEXT: call void (i64, ...) @continuation.continue(i64 2, i32 [[TMP2]], i64 ptrtoint (ptr @main.resume.0 to i64), [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I_FCA_0_INSERT]]), !continuation.registercount [[META20:![0-9]+]], !continuation.returnedRegistercount !20 ; SAVESTATE-NEXT: unreachable ; ; -; SAVESTATE-LABEL: define void @main.resume.0( -; SAVESTATE-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage !8 !continuation.registercount !20 !continuation !19 { +; SAVESTATE-LABEL: define dso_local void @main.resume.0( +; SAVESTATE-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage [[META8]] !continuation.registercount [[META20]] !continuation [[META19]] { ; SAVESTATE-NEXT: entryresume.0: ; SAVESTATE-NEXT: [[CONT_STATE:%.*]] = alloca [0 x i32], align 4 ; SAVESTATE-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(20) @PAYLOAD, align 4 ; SAVESTATE-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [1 x i32] poison, i32 [[TMP2]], 0 -; SAVESTATE-NEXT: [[DOTFCA_0_EXTRACT3:%.*]] = extractvalue [1 x i32] [[DOTFCA_0_INSERT]], 0 +; SAVESTATE-NEXT: [[DOTFCA_0_EXTRACT4:%.*]] = extractvalue [1 x i32] [[DOTFCA_0_INSERT]], 0 ; SAVESTATE-NEXT: [[DOTFCA_0_EXTRACT1:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], 0 ; SAVESTATE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; SAVESTATE-NEXT: call void @continuation.complete() ; SAVESTATE-NEXT: unreachable ; ; -; SAVESTATE-LABEL: define void @main_no_call() !lgc.rt.shaderstage !8 !continuation.entry !18 !continuation.registercount !8 !continuation !21 !continuation.state !8 { +; SAVESTATE-LABEL: define void @main_no_call( +; SAVESTATE-SAME: ) !lgc.rt.shaderstage [[META8]] !continuation.entry [[META18]] !continuation.registercount [[META8]] !continuation [[META21:![0-9]+]] !continuation.state [[META8]] { ; SAVESTATE-NEXT: AllocaSpillBB: ; SAVESTATE-NEXT: [[CONT_STATE:%.*]] = alloca [0 x i32], align 4 ; SAVESTATE-NEXT: [[TMP0:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA:%.*]] @continuations.getSystemData.s_struct.DispatchSystemDatas() @@ -337,7 +308,7 @@ attributes #2 = { nounwind } ; ; ; SAVESTATE-LABEL: define void @called( -; SAVESTATE-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !lgc.rt.shaderstage !22 !continuation.registercount !23 !continuation !24 !continuation.state !8 { +; SAVESTATE-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !lgc.rt.shaderstage [[META22:![0-9]+]] !continuation.registercount [[META23:![0-9]+]] !continuation [[META24:![0-9]+]] !continuation.state [[META8]] { ; SAVESTATE-NEXT: AllocaSpillBB: ; SAVESTATE-NEXT: [[CONT_STATE:%.*]] = alloca [0 x i32], align 4 ; SAVESTATE-NEXT: [[CSP:%.*]] = alloca i32, align 4 @@ -355,6 +326,6 @@ attributes #2 = { nounwind } ; SAVESTATE-NEXT: store i32 [[TMP4]], ptr addrspace(20) addrspacecast (ptr getelementptr (i32, ptr addrspacecast (ptr addrspace(20) @PAYLOAD to ptr), i64 2) to ptr addrspace(20)), align 4 ; SAVESTATE-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, i32 [[DOTFCA_0_EXTRACT]], 0 ; SAVESTATE-NEXT: [[TMP5:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP5]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount !23 +; SAVESTATE-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP5]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount [[META23]] ; SAVESTATE-NEXT: unreachable ; diff --git a/shared/continuations/test/dx/dxil-cont-convert-lgc-rt-op-trace-payload-type.ll b/shared/continuations/test/dx/dxil-cont-convert-lgc-rt-op-trace-payload-type.ll index b17cde3f96..5f4b4d1267 100644 --- a/shared/continuations/test/dx/dxil-cont-convert-lgc-rt-op-trace-payload-type.ll +++ b/shared/continuations/test/dx/dxil-cont-convert-lgc-rt-op-trace-payload-type.ll @@ -1,8 +1,8 @@ -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint' -S %s 2>%t0.stderr | FileCheck -check-prefix=PAYLOADTYPE-OPAQUE %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint' -S %s 2> %t0.stderr | FileCheck -check-prefix=PAYLOADTYPE-OPAQUE %s ; RUN: count 0 < %t0.stderr -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint' -S %s 2>%t1.stderr | FileCheck -check-prefix=PAYLOADTYPE2-OPAQUE %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint' -S %s 2> %t1.stderr | FileCheck -check-prefix=PAYLOADTYPE2-OPAQUE %s ; RUN: count 0 < %t1.stderr -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint' -S %s 2>%t2.stderr | FileCheck -check-prefix=PAYLOADTYPE3-OPAQUE %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint' -S %s 2> %t2.stderr | FileCheck -check-prefix=PAYLOADTYPE3-OPAQUE %s ; RUN: count 0 < %t2.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" diff --git a/shared/continuations/test/dx/dxil-cont-convert-lgc-rt-op-trace.ll b/shared/continuations/test/dx/dxil-cont-convert-lgc-rt-op-trace.ll index ab63417f82..c94258e1ca 100644 --- a/shared/continuations/test/dx/dxil-cont-convert-lgc-rt-op-trace.ll +++ b/shared/continuations/test/dx/dxil-cont-convert-lgc-rt-op-trace.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint' -S %s 2>%t0.stderr | FileCheck %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint' -S %s 2> %t0.stderr | FileCheck %s ; RUN: count 0 < %t0.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" diff --git a/shared/continuations/test/dx/dxil-cont-convert-lgc-rt-op.ll b/shared/continuations/test/dx/dxil-cont-convert-lgc-rt-op.ll index 0f68f4347f..cf7eba0c28 100644 --- a/shared/continuations/test/dx/dxil-cont-convert-lgc-rt-op.ll +++ b/shared/continuations/test/dx/dxil-cont-convert-lgc-rt-op.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function ClosestHit --version 3 -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,remove-types-metadata' -S %s 2>%t0.stderr | FileCheck %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,remove-types-metadata' -S %s 2> %t0.stderr | FileCheck %s ; RUN: count 0 < %t0.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" diff --git a/shared/continuations/test/dx/dxil-cont-intrinsic-prepare.ll b/shared/continuations/test/dx/dxil-cont-intrinsic-prepare.ll index fcd5257aba..74ec0f5fc5 100644 --- a/shared/continuations/test/dx/dxil-cont-intrinsic-prepare.ll +++ b/shared/continuations/test/dx/dxil-cont-intrinsic-prepare.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 3 -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint' -S %s 2>%t.stderr | FileCheck %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" diff --git a/shared/continuations/test/dx/dxil-cont-post-process-report-sizes.ll b/shared/continuations/test/dx/dxil-cont-post-process-report-sizes.ll index 0c9e4e3a63..ccb415c7e8 100644 --- a/shared/continuations/test/dx/dxil-cont-post-process-report-sizes.ll +++ b/shared/continuations/test/dx/dxil-cont-post-process-report-sizes.ll @@ -11,7 +11,7 @@ declare i32 @continuation.initialContinuationStackPtr() declare i32 @_cont_GetContinuationStackAddr() declare i32 @_cont_GetLocalRootIndex(%struct.DispatchSystemData*) declare %struct.DispatchSystemData @_cont_SetupRayGen() -declare void @continuation.continue() +declare void @continuation.continue(i64, ...) ; REPORT-CONT-SIZES: Continuation state size of "RayGen" (raygeneration): 108 bytes ; REPORT-PAYLOAD-SIZES: Incoming and max outgoing payload VGPR size of "RayGen" (raygeneration): 28 and 24 bytes @@ -19,7 +19,7 @@ define void @RayGen() !continuation.entry !0 !continuation !3 !continuation.stat %csp = alloca i32, align 4 %cspInit = call i32 @continuation.initialContinuationStackPtr() store i32 %cspInit, i32* %csp - call void @continuation.continue(), !continuation.registercount !6 + call void (i64, ...) @continuation.continue(i64 2), !continuation.registercount !6 ret void } @@ -33,7 +33,7 @@ define void @RayGen.resume.0(i32 %0, %struct.DispatchSystemData %1) !continuatio ; REPORT-PAYLOAD-SIZES: Incoming and max outgoing payload VGPR size of "CHS" (closesthit): 32 and 36 bytes ; REPORT-SYSTEM-DATA-SIZES-DAG: Incoming system data of "CHS" (closesthit) is "struct.CHSSystemData", size: 400 bytes define void @CHS(i32 %cspInit, i64 %returnAddr, %struct.CHSSystemData %0) !continuation !14 !continuation.registercount !8 !lgc.rt.shaderstage !13 { - call void @continuation.continue(), !continuation.registercount !9 + call void (i64, ...) @continuation.continue(i64 2), !continuation.registercount !9 ret void } diff --git a/shared/continuations/test/dx/dxil-cont-post-process.ll b/shared/continuations/test/dx/dxil-cont-post-process.ll index c8f826fca8..82fe3c7fa1 100644 --- a/shared/continuations/test/dx/dxil-cont-post-process.ll +++ b/shared/continuations/test/dx/dxil-cont-post-process.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt --verify-each -passes='dxil-cont-post-process,lint' -S %s 2>%t.stderr | FileCheck %s +; RUN: opt --verify-each -passes='dxil-cont-post-process,lint' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" diff --git a/shared/continuations/test/dx/dxil-cont-prepare-traversal.ll b/shared/continuations/test/dx/dxil-cont-prepare-traversal.ll index 384e985aed..efea1c0981 100644 --- a/shared/continuations/test/dx/dxil-cont-prepare-traversal.ll +++ b/shared/continuations/test/dx/dxil-cont-prepare-traversal.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 3 -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,remove-types-metadata' -S %s 2>%t0.stderr | FileCheck --check-prefix=PREPARE %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,remove-types-metadata' -S %s 2> %t0.stderr | FileCheck --check-prefix=PREPARE %s ; RUN: count 0 < %t0.stderr -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2>%t1.stderr | FileCheck --check-prefix=ALL %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2> %t1.stderr | FileCheck --check-prefix=ALL %s ; RUN: count 0 < %t1.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" @@ -25,7 +25,7 @@ declare !types !2 i32 @"\01?_AmdValueGetI32Something@@YA_KXZ"(%struct.TraversalD declare !types !3 void @"\01?_AmdValueSetI32Something@@YA_KXZ"(%struct.TraversalData*, i32, i32) ; Function Attrs: nounwind -define void @_AmdTraversal(i32 %stackPtr, %struct.TraversalData* %data) #0 !types !4 { +define void @_cont_Traversal(i32 %stackPtr, %struct.TraversalData* %data) #0 !types !4 { %1 = getelementptr inbounds %struct.TraversalData, %struct.TraversalData* %data, i32 0, i32 1 %2 = load i32, i32* %1, align 4 %3 = icmp eq i32 %2, 0 @@ -72,8 +72,8 @@ attributes #2 = { nounwind } !4 = !{!"function", !"void", i32 poison, !1} !5 = !{!"function", !"void", i64 poison, i64 poison, i32 poison, !6} !6 = !{i32 0, %struct.SystemData poison} -; PREPARE-LABEL: define void @_AmdTraversal( -; PREPARE-SAME: i32 [[STACKPTR:%.*]], [[STRUCT_TRAVERSALDATA:%.*]] [[DATA:%.*]]) #[[ATTR1:[0-9]+]] { +; PREPARE-LABEL: define void @_cont_Traversal( +; PREPARE-SAME: i32 [[STACKPTR:%.*]], [[STRUCT_TRAVERSALDATA:%.*]] [[DATA:%.*]]) #[[ATTR1:[0-9]+]] !lgc.rt.shaderstage [[META0:![0-9]+]] { ; PREPARE-NEXT: [[TMP1:%.*]] = alloca [[STRUCT_TRAVERSALDATA]], align 8 ; PREPARE-NEXT: store [[STRUCT_TRAVERSALDATA]] [[DATA]], ptr [[TMP1]], align 4 ; PREPARE-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[TMP1]], i32 0, i32 1 @@ -95,7 +95,7 @@ attributes #2 = { nounwind } ; PREPARE-NEXT: [[A4:%.*]] = add i32 [[A3]], [[TMP10]] ; PREPARE-NEXT: [[ADDR:%.*]] = zext i32 [[A4]] to i64 ; PREPARE-NEXT: [[TMP11:%.*]] = load [[STRUCT_SYSTEMDATA:%.*]], ptr [[TMP5]], align 4 -; PREPARE-NEXT: call void (i64, i64, ...) @continuation.waitContinue(i64 [[ADDR]], i64 -1, i32 [[STACKPTR]], i64 ptrtoint (ptr @_AmdTraversal to i64), [[STRUCT_SYSTEMDATA]] [[TMP11]]) +; PREPARE-NEXT: call void (i64, i64, ...) @continuation.waitContinue(i64 [[ADDR]], i64 -1, i32 [[STACKPTR]], i64 ptrtoint (ptr @_cont_Traversal to i64), [[STRUCT_SYSTEMDATA]] [[TMP11]]) ; PREPARE-NEXT: br label [[TMP14:%.*]] ; PREPARE: 12: ; PREPARE-NEXT: [[TMP13:%.*]] = load [[STRUCT_SYSTEMDATA]], ptr [[TMP5]], align 4 @@ -105,8 +105,8 @@ attributes #2 = { nounwind } ; PREPARE-NEXT: ret void ; ; -; ALL-LABEL: define void @_AmdTraversal( -; ALL-SAME: i32 [[STACKPTR:%.*]], [[STRUCT_TRAVERSALDATA:%.*]] [[DATA:%.*]]) #[[ATTR0:[0-9]+]] !continuation.registercount !0 { +; ALL-LABEL: define void @_cont_Traversal( +; ALL-SAME: i32 [[STACKPTR:%.*]], [[STRUCT_TRAVERSALDATA:%.*]] [[DATA:%.*]]) #[[ATTR0:[0-9]+]] !lgc.rt.shaderstage [[META2:![0-9]+]] !continuation [[META3:![0-9]+]] !continuation.registercount [[META0:![0-9]+]] { ; ALL-NEXT: [[TMP1:%.*]] = alloca [[STRUCT_TRAVERSALDATA]], align 8 ; ALL-NEXT: [[DATA_FCA_0_0_0_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[DATA]], 0, 0, 0 ; ALL-NEXT: [[DATA_FCA_0_0_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[TMP1]], i32 0, i32 0, i32 0, i32 0 @@ -141,8 +141,8 @@ attributes #2 = { nounwind } ; ALL-NEXT: [[DOTFCA_1_GEP:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[TMP5]], i32 0, i32 1 ; ALL-NEXT: [[DOTFCA_1_LOAD:%.*]] = load float, ptr [[DOTFCA_1_GEP]], align 4 ; ALL-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue [[STRUCT_SYSTEMDATA]] [[DOTFCA_0_0_INSERT]], float [[DOTFCA_1_LOAD]], 1 -; ALL-NEXT: [[TMP11:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @_AmdTraversal to i64)) -; ALL-NEXT: call void (i64, i64, ...) @continuation.waitContinue(i64 [[ADDR]], i64 -1, i32 [[STACKPTR]], i64 [[TMP11]], [[STRUCT_SYSTEMDATA]] [[DOTFCA_1_INSERT]]), !continuation.registercount !0 +; ALL-NEXT: [[TMP11:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @_cont_Traversal to i64)) +; ALL-NEXT: call void (i64, i64, ...) @continuation.waitContinue(i64 [[ADDR]], i64 -1, i32 [[STACKPTR]], i64 [[TMP11]], [[STRUCT_SYSTEMDATA]] [[DOTFCA_1_INSERT]]), !continuation.registercount [[META0]] ; ALL-NEXT: br label [[TMP13:%.*]] ; ALL: 12: ; ALL-NEXT: [[DOTFCA_0_0_GEP1:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[TMP5]], i32 0, i32 0, i32 0 @@ -151,7 +151,7 @@ attributes #2 = { nounwind } ; ALL-NEXT: [[DOTFCA_1_GEP4:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[TMP5]], i32 0, i32 1 ; ALL-NEXT: [[DOTFCA_1_LOAD5:%.*]] = load float, ptr [[DOTFCA_1_GEP4]], align 4 ; ALL-NEXT: [[DOTFCA_1_INSERT6:%.*]] = insertvalue [[STRUCT_SYSTEMDATA]] [[DOTFCA_0_0_INSERT3]], float [[DOTFCA_1_LOAD5]], 1 -; ALL-NEXT: call void (i64, i64, ...) @continuation.waitContinue(i64 0, i64 -1, i32 [[STACKPTR]], [[STRUCT_SYSTEMDATA]] [[DOTFCA_1_INSERT6]]), !continuation.registercount !0 +; ALL-NEXT: call void (i64, i64, ...) @continuation.waitContinue(i64 0, i64 -1, i32 [[STACKPTR]], [[STRUCT_SYSTEMDATA]] [[DOTFCA_1_INSERT6]]), !continuation.registercount [[META0]] ; ALL-NEXT: br label [[TMP13]] ; ALL: 13: ; ALL-NEXT: ret void diff --git a/shared/continuations/test/dx/global-mem-stack.ll b/shared/continuations/test/dx/global-mem-stack.ll index 9cd2857cc7..8eb7e1d3e7 100644 --- a/shared/continuations/test/dx/global-mem-stack.ll +++ b/shared/continuations/test/dx/global-mem-stack.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 3 -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2>%t.stderr | FileCheck %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" diff --git a/shared/continuations/test/dx/intersection-registercount.ll b/shared/continuations/test/dx/intersection-registercount.ll index aa78a035bf..a29a540f27 100644 --- a/shared/continuations/test/dx/intersection-registercount.ll +++ b/shared/continuations/test/dx/intersection-registercount.ll @@ -1,4 +1,4 @@ -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2>%t1.stderr | FileCheck %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2> %t1.stderr | FileCheck %s ; RUN: count 0 < %t1.stderr ; Check that the size of @REGISTERS is as big as the continuation.registercount when there is an intersection shader diff --git a/shared/continuations/test/dx/intrinsics/cont-payload-registers-get-i32.ll b/shared/continuations/test/dx/intrinsics/cont-payload-registers-get-i32.ll index 59a4ba2531..1ab6008549 100644 --- a/shared/continuations/test/dx/intrinsics/cont-payload-registers-get-i32.ll +++ b/shared/continuations/test/dx/intrinsics/cont-payload-registers-get-i32.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function main --version 2 -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2>%t.stderr | FileCheck %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr %struct.DispatchSystemData = type { i32 } diff --git a/shared/continuations/test/dx/intrinsics/cont-payload-registers-i32-count.ll b/shared/continuations/test/dx/intrinsics/cont-payload-registers-i32-count.ll index 4349c66c67..d08b814171 100644 --- a/shared/continuations/test/dx/intrinsics/cont-payload-registers-i32-count.ll +++ b/shared/continuations/test/dx/intrinsics/cont-payload-registers-i32-count.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 -; RUN: grep -v continuation.minPayloadRegisterCount %s | opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S 2>%t0.stderr | FileCheck -check-prefix=NOMINCOUNT %s +; RUN: grep -v continuation.minPayloadRegisterCount %s | opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S 2> %t0.stderr | FileCheck -check-prefix=NOMINCOUNT %s ; RUN: count 0 < %t0.stderr -; RUN: cat %s | opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S 2>%t1.stderr | FileCheck -check-prefix=MINCOUNT %s +; RUN: cat %s | opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S 2> %t1.stderr | FileCheck -check-prefix=MINCOUNT %s ; RUN: count 0 < %t1.stderr %struct.DispatchSystemData = type { i32 } diff --git a/shared/continuations/test/dx/intrinsics/cont-payload-registers-set-i32.ll b/shared/continuations/test/dx/intrinsics/cont-payload-registers-set-i32.ll index 36065b07f1..adbe62c135 100644 --- a/shared/continuations/test/dx/intrinsics/cont-payload-registers-set-i32.ll +++ b/shared/continuations/test/dx/intrinsics/cont-payload-registers-set-i32.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function main --version 2 -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2>%t.stderr | FileCheck %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr %struct.DispatchSystemData = type { i32 } diff --git a/shared/continuations/test/dx/intrinsics/cont-stack-alloc.ll b/shared/continuations/test/dx/intrinsics/cont-stack-alloc.ll index 3c70e7fe72..d8924d9f5b 100644 --- a/shared/continuations/test/dx/intrinsics/cont-stack-alloc.ll +++ b/shared/continuations/test/dx/intrinsics/cont-stack-alloc.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function main --version 2 -; RUN: opt --verify-each -passes='lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2>%t.stderr | FileCheck %s +; RUN: opt --verify-each -passes='cgscc(inline),lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr declare i32 @_AmdContStackAlloc(ptr %csp, i32 %size) diff --git a/shared/continuations/test/dx/intrinsics/continuation-stack-is-global-false.ll b/shared/continuations/test/dx/intrinsics/continuation-stack-is-global-false.ll index fbcfd0e4b6..b7625111be 100644 --- a/shared/continuations/test/dx/intrinsics/continuation-stack-is-global-false.ll +++ b/shared/continuations/test/dx/intrinsics/continuation-stack-is-global-false.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,dxil-cont-intrinsic-prepare,lint' -S %s 2>%t.stderr | FileCheck %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,dxil-cont-intrinsic-prepare,lint' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr %struct.DispatchSystemData = type { i32 } diff --git a/shared/continuations/test/dx/intrinsics/continuation-stack-is-global-true.ll b/shared/continuations/test/dx/intrinsics/continuation-stack-is-global-true.ll index 84e4a8e49b..1bd1dae523 100644 --- a/shared/continuations/test/dx/intrinsics/continuation-stack-is-global-true.ll +++ b/shared/continuations/test/dx/intrinsics/continuation-stack-is-global-true.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,dxil-cont-intrinsic-prepare,lint' -S %s 2>%t.stderr | FileCheck %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,dxil-cont-intrinsic-prepare,lint' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr %struct.DispatchSystemData = type { i32 } diff --git a/shared/continuations/test/dx/intrinsics/get-current-func-addr.ll b/shared/continuations/test/dx/intrinsics/get-current-func-addr.ll index bfa49545ec..7cd758b96c 100644 --- a/shared/continuations/test/dx/intrinsics/get-current-func-addr.ll +++ b/shared/continuations/test/dx/intrinsics/get-current-func-addr.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt --verify-each -passes='pre-coroutine-lowering,lint' -S %s 2>%t.stderr | FileCheck %s +; RUN: opt --verify-each -passes='lower-raytracing-pipeline,lint' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr declare void @Use(i64) diff --git a/shared/continuations/test/dx/intrinsics/get-flags.ll b/shared/continuations/test/dx/intrinsics/get-flags.ll index 5c2aafb0ff..22f40f57ae 100644 --- a/shared/continuations/test/dx/intrinsics/get-flags.ll +++ b/shared/continuations/test/dx/intrinsics/get-flags.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint' -S %s 2>%t.stderr | FileCheck %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr declare i32 @_AmdContinuationsGetFlags() diff --git a/shared/continuations/test/dx/intrinsics/get-rtip.ll b/shared/continuations/test/dx/intrinsics/get-rtip.ll index 51506d8251..b904103831 100644 --- a/shared/continuations/test/dx/intrinsics/get-rtip.ll +++ b/shared/continuations/test/dx/intrinsics/get-rtip.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint' -S %s 2>%t.stderr | FileCheck %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr declare i32 @_AmdGetRtip() diff --git a/shared/continuations/test/dx/intrinsics/get-shader-kind.ll b/shared/continuations/test/dx/intrinsics/get-shader-kind.ll index ccb840670c..95f52539fa 100644 --- a/shared/continuations/test/dx/intrinsics/get-shader-kind.ll +++ b/shared/continuations/test/dx/intrinsics/get-shader-kind.ll @@ -1,11 +1,19 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 -; RUN: opt --verify-each -passes='pre-coroutine-lowering,lint' -S %s 2>%t.stderr | FileCheck %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr +%struct.DispatchSystemData = type { i32 } +%struct.BuiltInTriangleIntersectionAttributes = type { <2 x float> } +%struct.HitData = type { float, i32 } %struct.Payload = type { i32 } declare i32 @_AmdGetShaderKind() +declare %struct.DispatchSystemData @_cont_SetupRayGen() +declare !types !3 i32 @_cont_GetLocalRootIndex(%struct.DispatchSystemData*) +declare !types !5 %struct.BuiltInTriangleIntersectionAttributes @_cont_GetTriangleHitAttributes(%struct.DispatchSystemData*) +declare !types !6 i32 @_cont_HitKind(%struct.DispatchSystemData*, %struct.HitData*) + ; Check that GetShaderKind calls in non-shaders, like left-over intrinsics, are ignored. define float @_cont_RayTCurrent() { ; CHECK-LABEL: define float @_cont_RayTCurrent() { @@ -19,12 +27,27 @@ define float @_cont_RayTCurrent() { } ; Note: DXILShaderKind::Miss has value 11 -define void @MyMiss(%struct.Payload* %payload) !lgc.rt.shaderstage !16 { -; CHECK-LABEL: define void @MyMiss -; CHECK-SAME: (ptr [[PAYLOAD:%.*]]) !lgc.rt.shaderstage !4 { -; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[STRUCT_PAYLOAD:%.*]], ptr [[PAYLOAD]], i32 0, i32 0 -; CHECK-NEXT: store i32 11, ptr [[TMP1]], align 4 -; CHECK-NEXT: ret void +define void @MyMiss(%struct.Payload* %payload) !types !1 !lgc.rt.shaderstage !16 { +; CHECK-LABEL: define %struct.DispatchSystemData @MyMiss +; CHECK-SAME: ([[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !types [[META12:![0-9]+]] !lgc.rt.shaderstage [[META14:![0-9]+]] !continuation.registercount [[META15:![0-9]+]] !continuation [[META16:![0-9]+]] { +; CHECK-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA]], align 8 +; CHECK-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_PAYLOAD:%.*]], align 8 +; CHECK-NEXT: [[TMP3:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() +; CHECK-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP3]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; CHECK-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[SYSTEM_DATA_ALLOCA]]) +; CHECK-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_PAYLOAD]], ptr [[TMP2]], i32 0, i32 0 +; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr @PAYLOAD, align 4 +; CHECK-NEXT: store i32 [[TMP5]], ptr [[TMP4]], align 4 +; CHECK-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_PAYLOAD]], ptr [[TMP2]], i32 0, i32 0 +; CHECK-NEXT: store i32 11, ptr [[TMP6]], align 4 +; CHECK-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_PAYLOAD]], ptr [[TMP2]], i32 0, i32 0 +; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4 +; CHECK-NEXT: store i32 [[TMP8]], ptr @PAYLOAD, align 4 +; CHECK-NEXT: [[TMP9:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; CHECK-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP9]], !continuation.registercount [[META15]] ; %1 = call i32 @_AmdGetShaderKind() %2 = getelementptr inbounds %struct.Payload, %struct.Payload* %payload, i32 0, i32 0 @@ -34,6 +57,13 @@ define void @MyMiss(%struct.Payload* %payload) !lgc.rt.shaderstage !16 { !dx.entryPoints = !{!12, !13} +!1 = !{!"function", !"void", !2} +!2 = !{i32 0, %struct.Payload poison} +!3 = !{!"function", i32 poison, !4} +!4 = !{i32 0, %struct.DispatchSystemData poison} +!5 = !{!"function", %struct.BuiltInTriangleIntersectionAttributes poison, !4} +!6 = !{!"function", i32 poison, !4, !7} +!7 = !{i32 0, %struct.HitData poison} !12 = !{null, !"", null, null, null} !13 = !{void (%struct.Payload*)* @MyMiss, !"MyMiss", null, null, !14} ; The metadata on this line identifies @MyMiss as miss shader diff --git a/shared/continuations/test/dx/intrinsics/shader-index.ll b/shared/continuations/test/dx/intrinsics/shader-index.ll index 3ad6dcba13..3819766269 100644 --- a/shared/continuations/test/dx/intrinsics/shader-index.ll +++ b/shared/continuations/test/dx/intrinsics/shader-index.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint' -S %s 2>%t.stderr | FileCheck %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr %struct.DispatchSystemData = type { i32 } @@ -14,16 +14,12 @@ declare %struct.DispatchSystemData @_cont_SetupRayGen() declare !types !8 i32 @_cont_GetLocalRootIndex(%struct.DispatchSystemData*) define i1 @_cont_ReportHit(%struct.DispatchSystemData* %data, float %t, i32 %hitKind) #0 !types !20 { -; CHECK-LABEL: define i1 @_cont_ReportHit( -; CHECK-SAME: ptr [[DATA:%.*]], float [[T:%.*]], i32 [[HITKIND:%.*]]) !types !13 { -; CHECK-NEXT: ret i1 true -; ret i1 true } define void @main() !lgc.rt.shaderstage !24 { ; CHECK-LABEL: define void @main( -; CHECK-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]]) !lgc.rt.shaderstage !14 !lgc.cps !14 !continuation !15 { +; CHECK-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]]) !lgc.rt.shaderstage [[META13:![0-9]+]] !lgc.cps [[META13]] !continuation [[META14:![0-9]+]] { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA:%.*]], align 8 ; CHECK-NEXT: [[TMP0:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() @@ -41,7 +37,7 @@ entry: define void @callable(%struct.Payload* %payload) !types !22 !lgc.rt.shaderstage !25 { ; CHECK-LABEL: define void @callable( -; CHECK-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !types !16 !lgc.rt.shaderstage !18 !lgc.cps !19 !continuation !20 { +; CHECK-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !types [[META15:![0-9]+]] !lgc.rt.shaderstage [[META17:![0-9]+]] !lgc.cps [[META18:![0-9]+]] !continuation [[META19:![0-9]+]] { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA]], align 8 ; CHECK-NEXT: [[TMP1:%.*]] = alloca [[STRUCT_PAYLOAD:%.*]], align 8 diff --git a/shared/continuations/test/dx/intrinsics/value-i32.ll b/shared/continuations/test/dx/intrinsics/value-i32.ll index 1afec2187d..9b82e871da 100644 --- a/shared/continuations/test/dx/intrinsics/value-i32.ll +++ b/shared/continuations/test/dx/intrinsics/value-i32.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 -; RUN: opt --verify-each -passes='dxil-cont-post-process,lint' -S %s 2>%t.stderr | FileCheck %s +; RUN: opt --verify-each -passes='dxil-cont-post-process,lint' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr %struct.Payload = type { float, i32, i64, i32 } diff --git a/shared/continuations/test/dx/lower-await.ll b/shared/continuations/test/dx/lower-await.ll index cbec15e004..23f44b23d4 100644 --- a/shared/continuations/test/dx/lower-await.ll +++ b/shared/continuations/test/dx/lower-await.ll @@ -1,9 +1,9 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt --verify-each -passes='lower-await,lint' -S %s 2>%t0.stderr | FileCheck -check-prefix=AWAIT %s +; RUN: opt --verify-each -passes='lower-await,lint' -S %s 2> %t0.stderr | FileCheck -check-prefix=AWAIT %s ; RUN: count 0 < %t0.stderr -; RUN: opt --verify-each -passes='lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint' -S %s 2>%t1.stderr | FileCheck -check-prefix=CORO %s +; RUN: opt --verify-each -passes='lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint' -S %s 2> %t1.stderr | FileCheck -check-prefix=CORO %s ; RUN: count 0 < %t1.stderr -; RUN: opt --verify-each -passes='lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint' -S %s 2>%t2.stderr | FileCheck -check-prefix=CLEANED %s +; RUN: opt --verify-each -passes='lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint' -S %s 2> %t2.stderr | FileCheck -check-prefix=CLEANED %s ; RUN: count 0 < %t2.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" @@ -37,27 +37,27 @@ define void @simple_await() !continuation.registercount !1 { ; CORO-NEXT: ret { ptr, ptr } [[TMP2]] ; ; CLEANED-LABEL: define void @simple_await( -; CLEANED-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]]) !continuation.registercount !2 !continuation !3 !continuation.state !4 !continuation.stacksize !4 { +; CLEANED-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]]) !continuation.registercount !1 !continuation !2 !continuation.state !3 !continuation.stacksize !3 { ; CLEANED-NEXT: AllocaSpillBB: ; CLEANED-NEXT: [[CONT_STATE:%.*]] = alloca [2 x i32], align 4 -; CLEANED-NEXT: call void @continuation.save.continuation_state() ; CLEANED-NEXT: [[RETURNADDR_SPILL_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; CLEANED-NEXT: store i64 [[RETURNADDR]], ptr [[RETURNADDR_SPILL_ADDR]], align 4 ; CLEANED-NEXT: [[TMP0:%.*]] = call ptr @continuation.getContinuationStackOffset() ; CLEANED-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4 ; CLEANED-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 8 ; CLEANED-NEXT: store i32 [[TMP2]], ptr [[TMP0]], align 4 -; CLEANED-NEXT: [[TMP3:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANED-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP3]]) +; CLEANED-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP1]] to ptr addrspace(21) ; CLEANED-NEXT: [[TMP4:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CLEANED-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4 -; CLEANED-NEXT: store i32 [[TMP5]], ptr @CONTINUATION_STATE, align 4 -; CLEANED-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CLEANED-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4 -; CLEANED-NEXT: store i32 [[TMP7]], ptr getelementptr inbounds ([2 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CLEANED-NEXT: [[TMP8:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANED-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4 -; CLEANED-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP9]], i64 ptrtoint (ptr @simple_await.resume.0 to i64)), !continuation.registercount !2, !continuation.returnedRegistercount !2 +; CLEANED-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 0 +; CLEANED-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4 +; CLEANED-NEXT: store i32 [[TMP6]], ptr addrspace(21) [[TMP5]], align 4 +; CLEANED-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CLEANED-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 1 +; CLEANED-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4 +; CLEANED-NEXT: store i32 [[TMP9]], ptr addrspace(21) [[TMP8]], align 4 +; CLEANED-NEXT: [[TMP10:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANED-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4 +; CLEANED-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP11]], i64 ptrtoint (ptr @simple_await.resume.0 to i64)), !continuation.registercount !1, !continuation.returnedRegistercount !1 ; CLEANED-NEXT: unreachable ; %tok = call %continuation.token* @async_fun(), !continuation.registercount !1, !continuation.returnedRegistercount !1 @@ -83,12 +83,12 @@ define void @simple_await_entry() !continuation.entry !0 !continuation.registerc ; CORO-NEXT: [[TMP2:%.*]] = insertvalue { ptr, ptr } [[TMP1]], ptr [[TOK]], 1 ; CORO-NEXT: ret { ptr, ptr } [[TMP2]] ; -; CLEANED-LABEL: define void @simple_await_entry() !continuation.registercount !2 !continuation.entry !5 !continuation !6 !continuation.state !2 { +; CLEANED-LABEL: define void @simple_await_entry() !continuation.registercount !1 !continuation.entry !4 !continuation !5 !continuation.state !1 { ; CLEANED-NEXT: AllocaSpillBB: ; CLEANED-NEXT: [[CONT_STATE:%.*]] = alloca [0 x i32], align 4 ; CLEANED-NEXT: [[TMP0:%.*]] = call ptr @continuation.getContinuationStackOffset() ; CLEANED-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4 -; CLEANED-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP1]], i64 ptrtoint (ptr @simple_await_entry.resume.0 to i64)), !continuation.registercount !2, !continuation.returnedRegistercount !2 +; CLEANED-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP1]], i64 ptrtoint (ptr @simple_await_entry.resume.0 to i64)), !continuation.registercount !1, !continuation.returnedRegistercount !1 ; CLEANED-NEXT: unreachable ; %tok = call %continuation.token* @async_fun(), !continuation.registercount !1, !continuation.returnedRegistercount !1 @@ -118,27 +118,27 @@ define void @await_with_arg(i32 %i) !continuation.registercount !1 { ; CORO-NEXT: ret { ptr, ptr } [[TMP2]] ; ; CLEANED-LABEL: define void @await_with_arg( -; CLEANED-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], i32 [[I:%.*]]) !continuation.registercount !2 !continuation !7 !continuation.state !4 !continuation.stacksize !4 { +; CLEANED-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], i32 [[I:%.*]]) !continuation.registercount !1 !continuation !6 !continuation.state !3 !continuation.stacksize !3 { ; CLEANED-NEXT: AllocaSpillBB: ; CLEANED-NEXT: [[CONT_STATE:%.*]] = alloca [2 x i32], align 4 -; CLEANED-NEXT: call void @continuation.save.continuation_state() ; CLEANED-NEXT: [[RETURNADDR_SPILL_ADDR:%.*]] = getelementptr inbounds [[AWAIT_WITH_ARG_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; CLEANED-NEXT: store i64 [[RETURNADDR]], ptr [[RETURNADDR_SPILL_ADDR]], align 4 ; CLEANED-NEXT: [[TMP0:%.*]] = call ptr @continuation.getContinuationStackOffset() ; CLEANED-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4 ; CLEANED-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 8 ; CLEANED-NEXT: store i32 [[TMP2]], ptr [[TMP0]], align 4 -; CLEANED-NEXT: [[TMP3:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANED-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP3]]) +; CLEANED-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP1]] to ptr addrspace(21) ; CLEANED-NEXT: [[TMP4:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CLEANED-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4 -; CLEANED-NEXT: store i32 [[TMP5]], ptr @CONTINUATION_STATE, align 4 -; CLEANED-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CLEANED-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4 -; CLEANED-NEXT: store i32 [[TMP7]], ptr getelementptr inbounds ([2 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CLEANED-NEXT: [[TMP8:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANED-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4 -; CLEANED-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun_with_arg to i64), i32 [[TMP9]], i64 ptrtoint (ptr @await_with_arg.resume.0 to i64), i32 [[I]]), !continuation.registercount !2, !continuation.returnedRegistercount !2 +; CLEANED-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 0 +; CLEANED-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4 +; CLEANED-NEXT: store i32 [[TMP6]], ptr addrspace(21) [[TMP5]], align 4 +; CLEANED-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CLEANED-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 1 +; CLEANED-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4 +; CLEANED-NEXT: store i32 [[TMP9]], ptr addrspace(21) [[TMP8]], align 4 +; CLEANED-NEXT: [[TMP10:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANED-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4 +; CLEANED-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun_with_arg to i64), i32 [[TMP11]], i64 ptrtoint (ptr @await_with_arg.resume.0 to i64), i32 [[I]]), !continuation.registercount !1, !continuation.returnedRegistercount !1 ; CLEANED-NEXT: unreachable ; %tok = call %continuation.token* @async_fun_with_arg(i32 %i), !continuation.registercount !1, !continuation.returnedRegistercount !1 @@ -168,27 +168,27 @@ define i32 @await_with_ret_value() !continuation.registercount !1 { ; CORO-NEXT: ret { ptr, ptr } [[TMP2]] ; ; CLEANED-LABEL: define void @await_with_ret_value( -; CLEANED-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]]) !continuation.registercount !2 !continuation !8 !continuation.state !4 !continuation.stacksize !4 { +; CLEANED-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]]) !continuation.registercount !1 !continuation !7 !continuation.state !3 !continuation.stacksize !3 { ; CLEANED-NEXT: AllocaSpillBB: ; CLEANED-NEXT: [[CONT_STATE:%.*]] = alloca [2 x i32], align 4 -; CLEANED-NEXT: call void @continuation.save.continuation_state() ; CLEANED-NEXT: [[RETURNADDR_SPILL_ADDR:%.*]] = getelementptr inbounds [[AWAIT_WITH_RET_VALUE_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; CLEANED-NEXT: store i64 [[RETURNADDR]], ptr [[RETURNADDR_SPILL_ADDR]], align 4 ; CLEANED-NEXT: [[TMP0:%.*]] = call ptr @continuation.getContinuationStackOffset() ; CLEANED-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4 ; CLEANED-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 8 ; CLEANED-NEXT: store i32 [[TMP2]], ptr [[TMP0]], align 4 -; CLEANED-NEXT: [[TMP3:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANED-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP3]]) +; CLEANED-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP1]] to ptr addrspace(21) ; CLEANED-NEXT: [[TMP4:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CLEANED-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4 -; CLEANED-NEXT: store i32 [[TMP5]], ptr @CONTINUATION_STATE, align 4 -; CLEANED-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CLEANED-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4 -; CLEANED-NEXT: store i32 [[TMP7]], ptr getelementptr inbounds ([2 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CLEANED-NEXT: [[TMP8:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANED-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4 -; CLEANED-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP9]], i64 ptrtoint (ptr @await_with_ret_value.resume.0 to i64)), !continuation.registercount !2, !continuation.returnedRegistercount !2 +; CLEANED-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 0 +; CLEANED-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4 +; CLEANED-NEXT: store i32 [[TMP6]], ptr addrspace(21) [[TMP5]], align 4 +; CLEANED-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CLEANED-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 1 +; CLEANED-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4 +; CLEANED-NEXT: store i32 [[TMP9]], ptr addrspace(21) [[TMP8]], align 4 +; CLEANED-NEXT: [[TMP10:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANED-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4 +; CLEANED-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64), i32 [[TMP11]], i64 ptrtoint (ptr @await_with_ret_value.resume.0 to i64)), !continuation.registercount !1, !continuation.returnedRegistercount !1 ; CLEANED-NEXT: unreachable ; %tok = call %continuation.token* @async_fun(), !continuation.registercount !1, !continuation.returnedRegistercount !1 @@ -217,27 +217,27 @@ define void @wait_await() !continuation.registercount !1 { ; CORO-NEXT: ret { ptr, ptr } [[TMP2]] ; ; CLEANED-LABEL: define void @wait_await( -; CLEANED-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]]) !continuation.registercount !2 !continuation !9 !continuation.state !4 !continuation.stacksize !4 { +; CLEANED-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]]) !continuation.registercount !1 !continuation !8 !continuation.state !3 !continuation.stacksize !3 { ; CLEANED-NEXT: AllocaSpillBB: ; CLEANED-NEXT: [[CONT_STATE:%.*]] = alloca [2 x i32], align 4 -; CLEANED-NEXT: call void @continuation.save.continuation_state() ; CLEANED-NEXT: [[RETURNADDR_SPILL_ADDR:%.*]] = getelementptr inbounds [[WAIT_AWAIT_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; CLEANED-NEXT: store i64 [[RETURNADDR]], ptr [[RETURNADDR_SPILL_ADDR]], align 4 ; CLEANED-NEXT: [[TMP0:%.*]] = call ptr @continuation.getContinuationStackOffset() ; CLEANED-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4 ; CLEANED-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 8 ; CLEANED-NEXT: store i32 [[TMP2]], ptr [[TMP0]], align 4 -; CLEANED-NEXT: [[TMP3:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANED-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP3]]) +; CLEANED-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP1]] to ptr addrspace(21) ; CLEANED-NEXT: [[TMP4:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CLEANED-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4 -; CLEANED-NEXT: store i32 [[TMP5]], ptr @CONTINUATION_STATE, align 4 -; CLEANED-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CLEANED-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4 -; CLEANED-NEXT: store i32 [[TMP7]], ptr getelementptr inbounds ([2 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CLEANED-NEXT: [[TMP8:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANED-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4 -; CLEANED-NEXT: call void (i64, i64, ...) @continuation.waitContinue(i64 ptrtoint (ptr @async_fun_with_waitmask to i64), i64 -1, i32 [[TMP9]], i64 ptrtoint (ptr @wait_await.resume.0 to i64)), !continuation.registercount !2, !continuation.returnedRegistercount !2 +; CLEANED-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 0 +; CLEANED-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP4]], align 4 +; CLEANED-NEXT: store i32 [[TMP6]], ptr addrspace(21) [[TMP5]], align 4 +; CLEANED-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CLEANED-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP3]], i32 0, i32 1 +; CLEANED-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4 +; CLEANED-NEXT: store i32 [[TMP9]], ptr addrspace(21) [[TMP8]], align 4 +; CLEANED-NEXT: [[TMP10:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANED-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4 +; CLEANED-NEXT: call void (i64, i64, ...) @continuation.waitContinue(i64 ptrtoint (ptr @async_fun_with_waitmask to i64), i64 -1, i32 [[TMP11]], i64 ptrtoint (ptr @wait_await.resume.0 to i64)), !continuation.registercount !1, !continuation.returnedRegistercount !1 ; CLEANED-NEXT: unreachable ; %tok = call %continuation.token* @async_fun_with_waitmask(i64 -1), !continuation.wait.await !0, !continuation.registercount !1, !continuation.returnedRegistercount !1 diff --git a/shared/continuations/test/dx/lower-rt-pipeline-call-shader.ll b/shared/continuations/test/dx/lower-rt-pipeline-call-shader.ll index 7c51f48076..e15c47304c 100644 --- a/shared/continuations/test/dx/lower-rt-pipeline-call-shader.ll +++ b/shared/continuations/test/dx/lower-rt-pipeline-call-shader.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 3 -; RUN: grep -v lgc.cps.module %s | opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S 2>%t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s +; RUN: grep -v lgc.cps.module %s | opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S 2> %t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s ; RUN: count 0 < %t0.stderr -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2>%t1.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE-CPS %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2> %t1.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE-CPS %s ; RUN: count 0 < %t1.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" @@ -74,84 +74,49 @@ attributes #0 = { nounwind } !18 = !{!"function", !"void", !14, i32 poison} !19 = !{!"function", !"void", i32 poison, i32 poison, !20} !20 = !{i32 0, %struct.TheirParams poison} -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_CallShader.struct.TheirParams( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i32 [[TMP0:%.*]], ptr [[TMP1:%.*]]) { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_THEIRPARAMS:%.*]], ptr [[TMP1]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP4]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = call ptr inttoptr (i64 2 to ptr)([[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]]), !continuation.registercount !16, !continuation.returnedRegistercount !16 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @await.struct.DispatchSystemData(ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_THEIRPARAMS]] poison, ptr [[TMP1]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_THEIRPARAMS]], ptr [[TMP1]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = load i32, ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP8]], ptr [[TMP7]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP6]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_CallShader( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i32 [[TMP0:%.*]]) { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[NEWDATA:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @_AmdAwaitShader(i64 2, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define void @main() !lgc.rt.shaderstage !9 !continuation.entry !17 !continuation.registercount !9 !continuation !18 { +; LOWERRAYTRACINGPIPELINE-LABEL: define void @main( +; LOWERRAYTRACINGPIPELINE-SAME: ) !lgc.rt.shaderstage [[META9:![0-9]+]] !continuation.entry [[META16:![0-9]+]] !continuation.registercount [[META9]] !continuation [[META17:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[PARAMS:%.*]] = alloca [[STRUCT_THEIRPARAMS:%.*]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP1:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[SYSTEM_DATA_ALLOCA]]) ; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_CallShader.struct.TheirParams(ptr [[SYSTEM_DATA_ALLOCA]], i32 1, ptr [[PARAMS]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void, !continuation.registercount !14 -; -; -; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define void @_cont_CallShader.struct.TheirParams( -; LOWERRAYTRACINGPIPELINE-CPS-SAME: ptr [[DATA:%.*]], i32 [[TMP0:%.*]], ptr [[TMP1:%.*]]) { -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_THEIRPARAMS:%.*]], ptr [[TMP1]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP4]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP5:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] (...) @lgc.cps.await.s_struct.DispatchSystemDatas(i32 2, i32 2, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_THEIRPARAMS]] poison, ptr [[TMP1]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_THEIRPARAMS]], ptr [[TMP1]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP7:%.*]] = load i32, ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP7]], ptr [[TMP6]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP5]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define void @_cont_CallShader( -; LOWERRAYTRACINGPIPELINE-CPS-SAME: ptr [[DATA:%.*]], i32 [[TMP0:%.*]]) { -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[NEWDATA:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @_AmdAwaitShader(i64 2, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: ret void +; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA_I:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_THEIRPARAMS]], ptr [[PARAMS]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP3]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = call ptr inttoptr (i64 2 to ptr)([[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I]]), !continuation.registercount [[META18:![0-9]+]], !continuation.returnedRegistercount !18 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @await.struct.DispatchSystemData(ptr [[TMP4]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_THEIRPARAMS]] poison, ptr [[PARAMS]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_THEIRPARAMS]], ptr [[PARAMS]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = load i32, ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP7]], ptr [[TMP6]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP5]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX1:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[SYSTEM_DATA_ALLOCA]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX1]]) +; LOWERRAYTRACINGPIPELINE-NEXT: ret void, !continuation.registercount [[META14:![0-9]+]] ; ; ; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define void @main( -; LOWERRAYTRACINGPIPELINE-CPS-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]]) !lgc.rt.shaderstage !9 !lgc.cps !9 !continuation !16 { +; LOWERRAYTRACINGPIPELINE-CPS-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]]) !lgc.rt.shaderstage [[META9:![0-9]+]] !lgc.cps [[META9]] !continuation [[META16:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[PARAMS:%.*]] = alloca [[STRUCT_THEIRPARAMS:%.*]], align 4 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP1:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[SYSTEM_DATA_ALLOCA]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_CallShader.struct.TheirParams(ptr [[SYSTEM_DATA_ALLOCA]], i32 1, ptr [[PARAMS]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[DIS_DATA_I:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_THEIRPARAMS]], ptr [[PARAMS]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP3]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP4:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] (...) @lgc.cps.await.s_struct.DispatchSystemDatas(i32 2, i32 2, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_THEIRPARAMS]] poison, ptr [[PARAMS]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_THEIRPARAMS]], ptr [[PARAMS]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP6:%.*]] = load i32, ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP6]], ptr [[TMP5]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP4]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX1:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[SYSTEM_DATA_ALLOCA]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX1]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: ret void ; diff --git a/shared/continuations/test/dx/lower-rt-pipeline-intrinsics-hit.ll b/shared/continuations/test/dx/lower-rt-pipeline-intrinsics-hit.ll index 05275edeee..9bf49097ee 100644 --- a/shared/continuations/test/dx/lower-rt-pipeline-intrinsics-hit.ll +++ b/shared/continuations/test/dx/lower-rt-pipeline-intrinsics-hit.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2>%t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2> %t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s ; RUN: count 0 < %t0.stderr -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2>%t1.stderr | FileCheck -check-prefix=DXILCONTPOSTPROCESS %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2> %t1.stderr | FileCheck -check-prefix=DXILCONTPOSTPROCESS %s ; RUN: count 0 < %t1.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" @@ -69,15 +69,6 @@ define i32 @_cont_GetLocalRootIndex(%struct.DispatchSystemData* %data) #0 !types } define void @_cont_TraceRay(%struct.DispatchSystemData* %data, i64 %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, float %6, float %7, float %8, float %9, float %10, float %11, float %12, float %13) #0 !types !35 { -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_TraceRay( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i64 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]], i32 [[TMP3:%.*]], i32 [[TMP4:%.*]], i32 [[TMP5:%.*]], float [[TMP6:%.*]], float [[TMP7:%.*]], float [[TMP8:%.*]], float [[TMP9:%.*]], float [[TMP10:%.*]], float [[TMP11:%.*]], float [[TMP12:%.*]], float [[TMP13:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @_cont_Traversal([[STRUCT_TRAVERSALDATA]] [[TRAV_DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP15]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; %dis_data = load %struct.DispatchSystemData, %struct.DispatchSystemData* %data, align 4 %sys_data = insertvalue %struct.SystemData undef, %struct.DispatchSystemData %dis_data, 0 %trav_data = insertvalue %struct.TraversalData undef, %struct.SystemData %sys_data, 0 @@ -87,13 +78,6 @@ define void @_cont_TraceRay(%struct.DispatchSystemData* %data, i64 %0, i32 %1, i } define i1 @_cont_ReportHit(%struct.AnyHitTraversalData* %data, float %t, i32 %hitKind) #0 !types !36 { -; LOWERRAYTRACINGPIPELINE-LABEL: define i1 @_cont_ReportHit( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], float [[T:%.*]], i32 [[HITKIND:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP1:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @_AmdAwaitAnyHit(i64 3, [[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA]], float [[T]], i32 [[HITKIND]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP1]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret i1 true -; %trav_data = load %struct.AnyHitTraversalData, %struct.AnyHitTraversalData* %data, align 4 %newdata = call %struct.AnyHitTraversalData @_AmdAwaitAnyHit(i64 3, %struct.AnyHitTraversalData %trav_data, float %t, i32 %hitKind) store %struct.AnyHitTraversalData %newdata, %struct.AnyHitTraversalData* %data, align 4 @@ -197,16 +181,15 @@ declare !types !46 [4 x <3 x float>] @_cont_WorldToObject4x3(%struct.DispatchSys ; Function Attrs: nounwind define void @RayGen() #3 { ; LOWERRAYTRACINGPIPELINE-LABEL: define void @RayGen( -; LOWERRAYTRACINGPIPELINE-SAME: ) #[[ATTR5:[0-9]+]] !continuation !28 !lgc.rt.shaderstage !19 !continuation.entry !14 !continuation.registercount !19 { +; LOWERRAYTRACINGPIPELINE-SAME: ) #[[ATTR5:[0-9]+]] !lgc.rt.shaderstage [[META19:![0-9]+]] !continuation [[META29:![0-9]+]] !continuation.entry [[META14:![0-9]+]] !continuation.registercount [[META19]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP1:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; LOWERRAYTRACINGPIPELINE-NEXT: ret void, !continuation.registercount [[META26:![0-9]+]] ; ; DXILCONTPOSTPROCESS-LABEL: define void @RayGen( -; DXILCONTPOSTPROCESS-SAME: ) #[[ATTR5:[0-9]+]] !continuation !27 !lgc.rt.shaderstage !18 !continuation.entry !13 !continuation.registercount !18 !continuation.state !18 { +; DXILCONTPOSTPROCESS-SAME: ) #[[ATTR5:[0-9]+]] !lgc.rt.shaderstage [[META18:![0-9]+]] !continuation [[META27:![0-9]+]] !continuation.entry [[META13:![0-9]+]] !continuation.registercount [[META18]] !continuation.state [[META18]] { ; DXILCONTPOSTPROCESS-NEXT: AllocaSpillBB: ; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA:%.*]], align 8 ; DXILCONTPOSTPROCESS-NEXT: [[CONT_STATE:%.*]] = alloca [0 x i32], align 4 @@ -224,41 +207,47 @@ define void @RayGen() #3 { ; Function Attrs: nounwind define void @Intersection() #3 { ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.AnyHitTraversalData @Intersection( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR5]] !continuation !29 !lgc.rt.shaderstage !30 !continuation.registercount !26 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR5]] !lgc.rt.shaderstage [[META30:![0-9]+]] !continuation [[META31:![0-9]+]] !continuation.registercount [[META26]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_HITDATA:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_HITDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_ANYHITTRAVERSALDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @continuations.getSystemData.s_struct.AnyHitTraversalDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP4]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = call float @lgc.rt.ray.tmin() ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[TMP8]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP9]], ptr [[TMP3]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = call float @_cont_RayTCurrent(ptr [[TMP7]], ptr [[TMP3]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = call i32 @lgc.rt.instance.id() -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[TMP13]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP14]], ptr [[TMP2]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = call i32 @_cont_HitKind(ptr [[TMP12]], ptr [[TMP2]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = call i1 @_cont_ReportHit.struct.BuiltInTriangleIntersectionAttributes(ptr [[SYSTEM_DATA_ALLOCA]], float 4.000000e+00, i32 0, ptr [[TMP16]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP19:%.*]] = call i1 @_cont_IsEndSearch(ptr [[TMP18]]) -; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[TMP19]], label [[TMP20:%.*]], label [[TMP22:%.*]] -; LOWERRAYTRACINGPIPELINE: 20: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP21:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP21]], !continuation.registercount !26 -; LOWERRAYTRACINGPIPELINE: 22: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP23:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP23]], !continuation.registercount !26 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_I:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA:%.*]], ptr [[TMP8]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I:%.*]] = load [[STRUCT_HITDATA]], ptr [[RESPTR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[RES_I]], ptr [[TMP3]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I1:%.*]] = load float, ptr [[TMP3]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = call i32 @lgc.rt.instance.id() +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_I2:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA]], ptr [[TMP11]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I3:%.*]] = load [[STRUCT_HITDATA]], ptr [[RESPTR_I2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[RES_I3]], ptr [[TMP2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_I4:%.*]] = getelementptr [[STRUCT_SYSTEMDATA:%.*]], ptr [[TMP10]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I5:%.*]] = load i32, ptr [[RESPTR_I4]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA_I:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP12]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = call ptr inttoptr (i64 3 to ptr)([[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA_I]], float 4.000000e+00, i32 0, [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP13]]), !continuation.registercount [[META26]], !continuation.returnedRegistercount !26 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @await.struct.AnyHitTraversalData(ptr [[TMP14]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP15]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[ISEND_I:%.*]] = call i1 @opaqueIsEnd() +; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[ISEND_I]], label [[TMP17:%.*]], label [[TMP19:%.*]] +; LOWERRAYTRACINGPIPELINE: 17: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP18]], !continuation.registercount [[META26]] +; LOWERRAYTRACINGPIPELINE: 19: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP20:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP20]], !continuation.registercount [[META26]] ; ; DXILCONTPOSTPROCESS-LABEL: define void @Intersection( -; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR5]] !continuation !28 !lgc.rt.shaderstage !29 !continuation.registercount !25 !continuation.state !30 !continuation.stacksize !30 { +; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR5]] !lgc.rt.shaderstage [[META28:![0-9]+]] !continuation [[META29:![0-9]+]] !continuation.registercount [[META25:![0-9]+]] !continuation.state [[META30:![0-9]+]] !continuation.stacksize [[META30]] { ; DXILCONTPOSTPROCESS-NEXT: AllocaSpillBB: ; DXILCONTPOSTPROCESS-NEXT: [[TMP1:%.*]] = alloca [[STRUCT_HITDATA:%.*]], align 8 ; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_ANYHITTRAVERSALDATA]], align 8 @@ -266,34 +255,31 @@ define void @Intersection() #3 { ; DXILCONTPOSTPROCESS-NEXT: [[CSP:%.*]] = alloca i32, align 4 ; DXILCONTPOSTPROCESS-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP0]], ptr [[SYSTEM_DATA]], align 4 ; DXILCONTPOSTPROCESS-NEXT: store i32 [[CSPINIT]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP2:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP3:%.*]] = add i32 [[TMP2]], 0 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP3]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP4:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(21) ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_SPILL_ADDR:%.*]] = getelementptr inbounds [[INTERSECTION_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: store i64 [[RETURNADDR]], ptr [[RETURNADDR_SPILL_ADDR]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP5:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_0_EXTRACT:%.*]] = extractvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP5]], 0, 0, 0, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_1_EXTRACT:%.*]] = extractvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP5]], 0, 0, 1 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_EXTRACT:%.*]] = extractvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP5]], 0, 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_1_EXTRACT:%.*]] = extractvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP5]], 0, 1, 1 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_2_EXTRACT:%.*]] = extractvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP5]], 0, 2 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_3_EXTRACT:%.*]] = extractvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP5]], 0, 3 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_4_EXTRACT:%.*]] = extractvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP5]], 0, 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_EXTRACT:%.*]] = extractvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP5]], 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_EXTRACT:%.*]] = extractvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP5]], 1, 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP2:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_0_EXTRACT:%.*]] = extractvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP2]], 0, 0, 0, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_1_EXTRACT:%.*]] = extractvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP2]], 0, 0, 1 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_EXTRACT:%.*]] = extractvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP2]], 0, 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_1_EXTRACT:%.*]] = extractvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP2]], 0, 1, 1 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_2_EXTRACT:%.*]] = extractvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP2]], 0, 2 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_3_EXTRACT:%.*]] = extractvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP2]], 0, 3 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_4_EXTRACT:%.*]] = extractvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP2]], 0, 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_EXTRACT:%.*]] = extractvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP2]], 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_EXTRACT:%.*]] = extractvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP2]], 1, 1 ; DXILCONTPOSTPROCESS-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) -; DXILCONTPOSTPROCESS-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA]], i32 0, i32 0, i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP7:%.*]] = call float @_cont_RayTMin(ptr [[TMP6]]) +; DXILCONTPOSTPROCESS-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA]], i32 0, i32 0, i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP4:%.*]] = call float @_cont_RayTMin(ptr [[TMP3]]) ; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] poison, float [[DOTFCA_0_1_0_EXTRACT]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_1_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] [[RES_I_FCA_0_INSERT]], i32 [[DOTFCA_0_1_1_EXTRACT]], 1 ; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_1_INSERT_FCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I_FCA_1_INSERT]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_1_INSERT_FCA_1_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I_FCA_1_INSERT]], 1 -; DXILCONTPOSTPROCESS-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA]], i32 0, i32 0, i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA]], i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP10:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[TMP9]]) -; DXILCONTPOSTPROCESS-NEXT: store [[STRUCT_HITDATA]] [[TMP10]], ptr [[TMP1]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP11:%.*]] = call i32 @_cont_InstanceID(ptr [[TMP8]], ptr [[TMP1]]) +; DXILCONTPOSTPROCESS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA]], i32 0, i32 0, i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[RESPTR_I:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA:%.*]], ptr [[TMP6]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I:%.*]] = load [[STRUCT_HITDATA]], ptr [[RESPTR_I]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store [[STRUCT_HITDATA]] [[RES_I]], ptr [[TMP1]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP7:%.*]] = call i32 @_cont_InstanceID(ptr [[TMP5]], ptr [[TMP1]]) ; DXILCONTPOSTPROCESS-NEXT: [[RES_I3_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] poison, float [[DOTFCA_0_1_0_EXTRACT]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[RES_I3_FCA_1_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] [[RES_I3_FCA_0_INSERT]], i32 [[DOTFCA_0_1_1_EXTRACT]], 1 ; DXILCONTPOSTPROCESS-NEXT: [[RES_I3_FCA_1_INSERT_FCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I3_FCA_1_INSERT]], 0 @@ -308,27 +294,21 @@ define void @Intersection() #3 { ; DXILCONTPOSTPROCESS-NEXT: [[TRAV_DATA_I_FCA_1_0_INSERT:%.*]] = insertvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA_I_FCA_0_4_INSERT]], float [[DOTFCA_1_0_EXTRACT]], 1, 0 ; DXILCONTPOSTPROCESS-NEXT: [[TRAV_DATA_I_FCA_1_1_INSERT:%.*]] = insertvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA_I_FCA_1_0_INSERT]], i32 [[DOTFCA_1_1_EXTRACT]], 1, 1 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] poison, <2 x float> undef, 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP12:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP13:%.*]] = add i32 [[TMP12]], 8 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP13]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP16:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP17:%.*]] = inttoptr i32 [[TMP16]] to ptr addrspace(21) -; DXILCONTPOSTPROCESS-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP17]], i64 -8 -; DXILCONTPOSTPROCESS-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP18]], i32 0 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP15]], ptr addrspace(21) [[TMP19]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP20:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP22:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP23:%.*]] = inttoptr i32 [[TMP22]] to ptr addrspace(21) -; DXILCONTPOSTPROCESS-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP23]], i64 -8 -; DXILCONTPOSTPROCESS-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP24]], i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP26:%.*]] = getelementptr [2 x i32], ptr addrspace(21) [[TMP25]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP21]], ptr addrspace(21) [[TMP26]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP27:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP28:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @Intersection.resume.0 to i64)) -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 3, i32 [[TMP27]], i64 [[TMP28]], [[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA_I_FCA_1_1_INSERT]], float 4.000000e+00, i32 0, [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[DOTFCA_0_INSERT]]), !continuation.registercount !25, !continuation.returnedRegistercount !25 +; DXILCONTPOSTPROCESS-NEXT: [[TMP8:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP9:%.*]] = add i32 [[TMP8]], 8 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP9]], ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP10:%.*]] = inttoptr i32 [[TMP8]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP11:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP10]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP11]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP13]], ptr addrspace(21) [[TMP12]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP15:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP10]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP14]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP16]], ptr addrspace(21) [[TMP15]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP17:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP18:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @Intersection.resume.0 to i64)) +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 3, i32 [[TMP17]], i64 [[TMP18]], [[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA_I_FCA_1_1_INSERT]], float 4.000000e+00, i32 0, [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[DOTFCA_0_INSERT]]), !continuation.registercount [[META25]], !continuation.returnedRegistercount !25 ; DXILCONTPOSTPROCESS-NEXT: unreachable ; %1 = call float @dx.op.rayTMin.f32(i32 153) @@ -343,7 +323,7 @@ define void @Intersection() #3 { ; Function Attrs: nounwind define void @AnyHit(%struct.RayPayload* noalias nocapture %payload, %struct.BuiltInTriangleIntersectionAttributes* nocapture readonly %attr) #3 !types !47 { ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.AnyHitTraversalData @AnyHit( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] [[TMP1:%.*]]) #[[ATTR5]] !continuation !31 !lgc.rt.shaderstage !32 !continuation.registercount !33 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] [[TMP1:%.*]]) #[[ATTR5]] !lgc.rt.shaderstage [[META32:![0-9]+]] !continuation [[META33:![0-9]+]] !continuation.registercount [[META34:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_HITDATA:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = alloca [[STRUCT_HITDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = alloca [[STRUCT_HITDATA]], align 8 @@ -356,8 +336,7 @@ define void @AnyHit(%struct.RayPayload* noalias nocapture %payload, %struct.Buil ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @continuations.getSystemData.s_struct.AnyHitTraversalDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP9]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP10]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP8]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = load float, ptr @PAYLOAD, align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP12]], ptr [[TMP11]], align 4 @@ -387,54 +366,58 @@ define void @AnyHit(%struct.RayPayload* noalias nocapture %payload, %struct.Buil ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = call float @_cont_RayTMin(ptr [[TMP27]]) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP30:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP31:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[TMP30]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP31]], ptr [[TMP5]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP32:%.*]] = call float @_cont_RayTCurrent(ptr [[TMP29]], ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP33:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP35:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[TMP34]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP35]], ptr [[TMP3]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP36:%.*]] = call i32 @_cont_InstanceID(ptr [[TMP33]], ptr [[TMP3]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP37:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[TMP38]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP39]], ptr [[TMP4]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP40:%.*]] = call i32 @_cont_HitKind(ptr [[TMP37]], ptr [[TMP4]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP41:%.*]] = insertvalue [[STRUCT_RAYPAYLOAD]] undef, float [[TMP28]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP42:%.*]] = insertvalue [[STRUCT_RAYPAYLOAD]] [[TMP41]], float [[TMP32]], 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP43:%.*]] = insertvalue [[STRUCT_RAYPAYLOAD]] [[TMP42]], i32 [[TMP36]], 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP44:%.*]] = insertvalue [[STRUCT_RAYPAYLOAD]] [[TMP43]], i32 [[TMP40]], 3 -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_RAYPAYLOAD]] [[TMP44]], ptr [[TMP8]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_I:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA:%.*]], ptr [[TMP30]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I:%.*]] = load [[STRUCT_HITDATA]], ptr [[RESPTR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[RES_I]], ptr [[TMP5]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I1:%.*]] = load float, ptr [[TMP5]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP31:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP32:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_I6:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA]], ptr [[TMP32]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I7:%.*]] = load [[STRUCT_HITDATA]], ptr [[RESPTR_I6]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[RES_I7]], ptr [[TMP3]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP33:%.*]] = call i32 @_cont_InstanceID(ptr [[TMP31]], ptr [[TMP3]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP35:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_I2:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA]], ptr [[TMP35]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I3:%.*]] = load [[STRUCT_HITDATA]], ptr [[RESPTR_I2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[RES_I3]], ptr [[TMP4]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_I4:%.*]] = getelementptr [[STRUCT_SYSTEMDATA:%.*]], ptr [[TMP34]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I5:%.*]] = load i32, ptr [[RESPTR_I4]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP36:%.*]] = insertvalue [[STRUCT_RAYPAYLOAD]] undef, float [[TMP28]], 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP37:%.*]] = insertvalue [[STRUCT_RAYPAYLOAD]] [[TMP36]], float [[RES_I1]], 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = insertvalue [[STRUCT_RAYPAYLOAD]] [[TMP37]], i32 [[TMP33]], 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = insertvalue [[STRUCT_RAYPAYLOAD]] [[TMP38]], i32 [[RES_I5]], 3 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_RAYPAYLOAD]] [[TMP39]], ptr [[TMP8]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_AcceptHit(ptr [[SYSTEM_DATA_ALLOCA]]) ; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP45:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP8]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP46:%.*]] = load float, ptr [[TMP45]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP46]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP47:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP8]], i32 0, i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP48:%.*]] = load float, ptr [[TMP47]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP48]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP49:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP8]], i32 0, i32 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP50:%.*]] = load i32, ptr [[TMP49]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP50]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i32 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP51:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP8]], i32 0, i32 3 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP52:%.*]] = load i32, ptr [[TMP51]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP52]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i32 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP53:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP54:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP55:%.*]] = load i32, ptr [[TMP53]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP55]], ptr [[TMP54]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP56:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP57:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP58:%.*]] = load i32, ptr [[TMP56]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP58]], ptr [[TMP57]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP59:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP6]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP60:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP60]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP59]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP61:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP61]], !continuation.registercount !33 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP40:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP8]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP41:%.*]] = load float, ptr [[TMP40]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP41]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP42:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP8]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP43:%.*]] = load float, ptr [[TMP42]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP43]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP44:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP8]], i32 0, i32 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP45:%.*]] = load i32, ptr [[TMP44]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP45]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i32 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP46:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP8]], i32 0, i32 3 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP47:%.*]] = load i32, ptr [[TMP46]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP47]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i32 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP48:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP49:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP50:%.*]] = load i32, ptr [[TMP48]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP50]], ptr [[TMP49]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP51:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP52:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP53:%.*]] = load i32, ptr [[TMP51]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP53]], ptr [[TMP52]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP54:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP6]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP55:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP55]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP54]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP56:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP56]], !continuation.registercount [[META34]] ; ; DXILCONTPOSTPROCESS-LABEL: define void @AnyHit( -; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] [[TMP1:%.*]]) #[[ATTR5]] !continuation !31 !lgc.rt.shaderstage !32 !continuation.registercount !33 !continuation.state !18 { +; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] [[TMP1:%.*]]) #[[ATTR5]] !lgc.rt.shaderstage [[META31:![0-9]+]] !continuation [[META32:![0-9]+]] !continuation.registercount [[META33:![0-9]+]] !continuation.state [[META18]] { ; DXILCONTPOSTPROCESS-NEXT: AllocaSpillBB: ; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_ANYHITTRAVERSALDATA]], align 8 ; DXILCONTPOSTPROCESS-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_HITDATA:%.*]], align 8 @@ -500,7 +483,23 @@ define void @AnyHit(%struct.RayPayload* noalias nocapture %payload, %struct.Buil ; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_1_INSERT_FCA_1_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I_FCA_1_INSERT]], 1 ; DXILCONTPOSTPROCESS-NEXT: [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: [[TMP18:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[RESPTR_I2:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA]], ptr [[TMP18]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[RESPTR_I6:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA]], ptr [[TMP18]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I7_FCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[RESPTR_I6]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I7_FCA_0_LOAD:%.*]] = load float, ptr [[RES_I7_FCA_0_GEP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I7_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] poison, float [[RES_I7_FCA_0_LOAD]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I7_FCA_1_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[RESPTR_I6]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I7_FCA_1_LOAD:%.*]] = load i32, ptr [[RES_I7_FCA_1_GEP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I7_FCA_1_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] [[RES_I7_FCA_0_INSERT]], i32 [[RES_I7_FCA_1_LOAD]], 1 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I7_FCA_1_INSERT_FCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I7_FCA_1_INSERT]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I7_FCA_1_INSERT_FCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[TMP2]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: store float [[RES_I7_FCA_1_INSERT_FCA_0_EXTRACT]], ptr [[RES_I7_FCA_1_INSERT_FCA_0_GEP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I7_FCA_1_INSERT_FCA_1_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I7_FCA_1_INSERT]], 1 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I7_FCA_1_INSERT_FCA_1_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[TMP2]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[RES_I7_FCA_1_INSERT_FCA_1_EXTRACT]], ptr [[RES_I7_FCA_1_INSERT_FCA_1_GEP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP19:%.*]] = call i32 @_cont_InstanceID(ptr [[TMP17]], ptr [[TMP2]]) +; DXILCONTPOSTPROCESS-NEXT: [[TMP20:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[RESPTR_I2:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA]], ptr [[TMP21]], i32 0, i32 1 ; DXILCONTPOSTPROCESS-NEXT: [[RES_I3_FCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[RESPTR_I2]], i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: [[RES_I3_FCA_0_LOAD:%.*]] = load float, ptr [[RES_I3_FCA_0_GEP]], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[RES_I3_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] poison, float [[RES_I3_FCA_0_LOAD]], 0 @@ -508,29 +507,13 @@ define void @AnyHit(%struct.RayPayload* noalias nocapture %payload, %struct.Buil ; DXILCONTPOSTPROCESS-NEXT: [[RES_I3_FCA_1_LOAD:%.*]] = load i32, ptr [[RES_I3_FCA_1_GEP]], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[RES_I3_FCA_1_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] [[RES_I3_FCA_0_INSERT]], i32 [[RES_I3_FCA_1_LOAD]], 1 ; DXILCONTPOSTPROCESS-NEXT: [[RES_I3_FCA_1_INSERT_FCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I3_FCA_1_INSERT]], 0 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I3_FCA_1_INSERT_FCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[TMP2]], i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: store float [[RES_I3_FCA_1_INSERT_FCA_0_EXTRACT]], ptr [[RES_I3_FCA_1_INSERT_FCA_0_GEP]], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[RES_I3_FCA_1_INSERT_FCA_1_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I3_FCA_1_INSERT]], 1 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I3_FCA_1_INSERT_FCA_1_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[TMP2]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[RES_I3_FCA_1_INSERT_FCA_1_EXTRACT]], ptr [[RES_I3_FCA_1_INSERT_FCA_1_GEP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP19:%.*]] = call i32 @_cont_InstanceID(ptr [[TMP17]], ptr [[TMP2]]) -; DXILCONTPOSTPROCESS-NEXT: [[TMP20:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[RESPTR_I4:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA]], ptr [[TMP21]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I5_FCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[RESPTR_I4]], i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I5_FCA_0_LOAD:%.*]] = load float, ptr [[RES_I5_FCA_0_GEP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I5_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] poison, float [[RES_I5_FCA_0_LOAD]], 0 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I5_FCA_1_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[RESPTR_I4]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I5_FCA_1_LOAD:%.*]] = load i32, ptr [[RES_I5_FCA_1_GEP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I5_FCA_1_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] [[RES_I5_FCA_0_INSERT]], i32 [[RES_I5_FCA_1_LOAD]], 1 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I5_FCA_1_INSERT_FCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I5_FCA_1_INSERT]], 0 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I5_FCA_1_INSERT_FCA_1_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I5_FCA_1_INSERT]], 1 -; DXILCONTPOSTPROCESS-NEXT: [[RESPTR_I6:%.*]] = getelementptr [[STRUCT_SYSTEMDATA:%.*]], ptr [[TMP20]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I7:%.*]] = load i32, ptr [[RESPTR_I6]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[RESPTR_I4:%.*]] = getelementptr [[STRUCT_SYSTEMDATA:%.*]], ptr [[TMP20]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I5:%.*]] = load i32, ptr [[RESPTR_I4]], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[TMP22:%.*]] = insertvalue [[STRUCT_RAYPAYLOAD:%.*]] undef, float [[TMP14]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[TMP23:%.*]] = insertvalue [[STRUCT_RAYPAYLOAD]] [[TMP22]], float [[RES_I_FCA_1_INSERT_FCA_0_EXTRACT]], 1 ; DXILCONTPOSTPROCESS-NEXT: [[TMP24:%.*]] = insertvalue [[STRUCT_RAYPAYLOAD]] [[TMP23]], i32 [[TMP19]], 2 -; DXILCONTPOSTPROCESS-NEXT: [[TMP25:%.*]] = insertvalue [[STRUCT_RAYPAYLOAD]] [[TMP24]], i32 [[RES_I7]], 3 +; DXILCONTPOSTPROCESS-NEXT: [[TMP25:%.*]] = insertvalue [[STRUCT_RAYPAYLOAD]] [[TMP24]], i32 [[RES_I5]], 3 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_EXTRACT8:%.*]] = extractvalue [[STRUCT_RAYPAYLOAD]] [[TMP25]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_EXTRACT:%.*]] = extractvalue [[STRUCT_RAYPAYLOAD]] [[TMP25]], 1 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_EXTRACT:%.*]] = extractvalue [[STRUCT_RAYPAYLOAD]] [[TMP25]], 2 @@ -579,7 +562,7 @@ define void @AnyHit(%struct.RayPayload* noalias nocapture %payload, %struct.Buil ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_LOAD:%.*]] = load i32, ptr [[DOTFCA_1_1_GEP17]], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_INSERT:%.*]] = insertvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[DOTFCA_1_0_INSERT]], i32 [[DOTFCA_1_1_LOAD]], 1, 1 ; DXILCONTPOSTPROCESS-NEXT: [[TMP31:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP31]], [[STRUCT_ANYHITTRAVERSALDATA]] [[DOTFCA_1_1_INSERT]]), !continuation.registercount !33 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP31]], [[STRUCT_ANYHITTRAVERSALDATA]] [[DOTFCA_1_1_INSERT]]), !continuation.registercount [[META33]] ; DXILCONTPOSTPROCESS-NEXT: unreachable ; %1 = call float @dx.op.rayTMin.f32(i32 153) @@ -597,7 +580,7 @@ define void @AnyHit(%struct.RayPayload* noalias nocapture %payload, %struct.Buil ; Function Attrs: nounwind define void @ClosestHit(%struct.RayPayload* noalias nocapture %payload, %struct.BuiltInTriangleIntersectionAttributes* nocapture readonly %attr) #3 !types !47 { ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.DispatchSystemData @ClosestHit( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR5]] !continuation !34 !lgc.rt.shaderstage !35 !continuation.registercount !33 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR5]] !lgc.rt.shaderstage [[META35:![0-9]+]] !continuation [[META36:![0-9]+]] !continuation.registercount [[META34]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_HITDATA:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_HITDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = alloca [[STRUCT_HITDATA]], align 8 @@ -608,8 +591,7 @@ define void @ClosestHit(%struct.RayPayload* noalias nocapture %payload, %struct. ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = call [[STRUCT_SYSTEMDATA]] @continuations.getSystemData.s_struct.SystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_SYSTEMDATA]] [[TMP7]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP8]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP6]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = load float, ptr @PAYLOAD, align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP10]], ptr [[TMP9]], align 4 @@ -638,33 +620,34 @@ define void @ClosestHit(%struct.RayPayload* noalias nocapture %payload, %struct. ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP26:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP27:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCommittedState(ptr [[SYSTEM_DATA_ALLOCA]]) ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP27]], ptr [[TMP4]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = call float @_cont_RayTCurrent(ptr [[TMP26]], ptr [[TMP4]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP30:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCommittedState(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP30]], ptr [[TMP2]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP31:%.*]] = call i32 @_cont_InstanceID(ptr [[TMP29]], ptr [[TMP2]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP32:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCommittedState(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP32]], ptr [[TMP3]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP33:%.*]] = call i32 @_cont_HitKind(ptr [[SYSTEM_DATA_ALLOCA]], ptr [[TMP3]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I:%.*]] = load float, ptr [[TMP4]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCommittedState(ptr [[SYSTEM_DATA_ALLOCA]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP29]], ptr [[TMP2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP30:%.*]] = call i32 @_cont_InstanceID(ptr [[TMP28]], ptr [[TMP2]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP31:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCommittedState(ptr [[SYSTEM_DATA_ALLOCA]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP31]], ptr [[TMP3]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_I:%.*]] = getelementptr [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I1:%.*]] = load i32, ptr [[RESPTR_I]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP6]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP32:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP6]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP33:%.*]] = load float, ptr [[TMP32]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP33]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP6]], i32 0, i32 1 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP35:%.*]] = load float, ptr [[TMP34]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP35]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP36:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP6]], i32 0, i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP37:%.*]] = load float, ptr [[TMP36]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP37]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP6]], i32 0, i32 2 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP35]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP36:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP6]], i32 0, i32 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP37:%.*]] = load i32, ptr [[TMP36]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP37]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i32 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP6]], i32 0, i32 3 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = load i32, ptr [[TMP38]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP39]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i32 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP40:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP6]], i32 0, i32 3 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP41:%.*]] = load i32, ptr [[TMP40]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP41]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i32 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP42:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP43:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP42]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP43]], !continuation.registercount !33 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP39]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i32 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP40:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP41:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP40]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP41]], !continuation.registercount [[META34]] ; ; DXILCONTPOSTPROCESS-LABEL: define void @ClosestHit( -; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR5]] !continuation !34 !lgc.rt.shaderstage !35 !continuation.registercount !33 !continuation.state !18 { +; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR5]] !lgc.rt.shaderstage [[META34:![0-9]+]] !continuation [[META35:![0-9]+]] !continuation.registercount [[META33]] !continuation.state [[META18]] { ; DXILCONTPOSTPROCESS-NEXT: AllocaSpillBB: ; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 ; DXILCONTPOSTPROCESS-NEXT: [[TMP1:%.*]] = alloca [[STRUCT_HITDATA:%.*]], align 8 @@ -721,7 +704,7 @@ define void @ClosestHit(%struct.RayPayload* noalias nocapture %payload, %struct. ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_LOAD:%.*]] = load <3 x i32>, ptr [[DOTFCA_0_GEP]], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, <3 x i32> [[DOTFCA_0_LOAD]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[TMP20:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP20]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount !33 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP20]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount [[META33]] ; DXILCONTPOSTPROCESS-NEXT: unreachable ; %1 = call float @dx.op.rayTMin.f32(i32 153) diff --git a/shared/continuations/test/dx/lower-rt-pipeline-intrinsics.ll b/shared/continuations/test/dx/lower-rt-pipeline-intrinsics.ll index 58af828425..24fe7e686c 100644 --- a/shared/continuations/test/dx/lower-rt-pipeline-intrinsics.ll +++ b/shared/continuations/test/dx/lower-rt-pipeline-intrinsics.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function ClosestHit --version 3 -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2>%t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2> %t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s ; RUN: count 0 < %t0.stderr -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2>%t1.stderr | FileCheck -check-prefix=DXILCONTPOSTPROCESS %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2> %t1.stderr | FileCheck -check-prefix=DXILCONTPOSTPROCESS %s ; RUN: count 0 < %t1.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" @@ -97,7 +97,7 @@ declare !types !36 i32 @_cont_HitKind(%struct.SystemData* nocapture readnone, %s ; Function Attrs: nounwind define void @ClosestHit(%struct.RayPayload* %0, %struct.BuiltInTriangleIntersectionAttributes* %1) #3 !types !37 { ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.DispatchSystemData @ClosestHit( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR3:[0-9]+]] !continuation !22 !lgc.rt.shaderstage !23 !continuation.registercount !24 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR3:[0-9]+]] !lgc.rt.shaderstage [[META23:![0-9]+]] !continuation [[META24:![0-9]+]] !continuation.registercount [[META25:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_HITDATA:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_HITDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = alloca [[STRUCT_HITDATA]], align 8 @@ -116,8 +116,7 @@ define void @ClosestHit(%struct.RayPayload* %0, %struct.BuiltInTriangleIntersect ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = call [[STRUCT_SYSTEMDATA]] @continuations.getSystemData.s_struct.SystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_SYSTEMDATA]] [[TMP15]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP16]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP14]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = load float, ptr @PAYLOAD, align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP18]], ptr [[TMP17]], align 4 @@ -215,10 +214,10 @@ define void @ClosestHit(%struct.RayPayload* %0, %struct.BuiltInTriangleIntersect ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP75]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i32 9), align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP76:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP77:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP76]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP77]], !continuation.registercount !24 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP77]], !continuation.registercount [[META25]] ; ; DXILCONTPOSTPROCESS-LABEL: define void @ClosestHit( -; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR3:[0-9]+]] !continuation !21 !lgc.rt.shaderstage !22 !continuation.registercount !23 !continuation.state !18 { +; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR3:[0-9]+]] !lgc.rt.shaderstage [[META21:![0-9]+]] !continuation [[META22:![0-9]+]] !continuation.registercount [[META23:![0-9]+]] !continuation.state [[META18:![0-9]+]] { ; DXILCONTPOSTPROCESS-NEXT: AllocaSpillBB: ; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 ; DXILCONTPOSTPROCESS-NEXT: [[TMP1:%.*]] = alloca [[STRUCT_HITDATA:%.*]], align 8 @@ -368,7 +367,7 @@ define void @ClosestHit(%struct.RayPayload* %0, %struct.BuiltInTriangleIntersect ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_LOAD:%.*]] = load <3 x i32>, ptr [[DOTFCA_0_GEP]], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, <3 x i32> [[DOTFCA_0_LOAD]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[TMP58:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP58]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount !23 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP58]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount [[META23]] ; DXILCONTPOSTPROCESS-NEXT: unreachable ; %a = call i32 @dx.op.dispatchRaysIndex.i32(i32 145, i8 0) diff --git a/shared/continuations/test/dx/lower-rt-pipeline-large-payload.ll b/shared/continuations/test/dx/lower-rt-pipeline-large-payload.ll new file mode 100644 index 0000000000..387ef66b41 --- /dev/null +++ b/shared/continuations/test/dx/lower-rt-pipeline-large-payload.ll @@ -0,0 +1,598 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 3 +; Test handling of large payloads and payload spilling. +; We set the max number of payload registers to 2, so relatively small payloads need to spill already. +; This results in a bit nicer result IR, containing less "spam" copying payload fields around. +; We also set a max hit attribute size ensuring there is no need for hit attribute storage in the payload. +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2> %t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s +; RUN: count 0 < %t0.stderr +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2> %t1.stderr | FileCheck -check-prefix=DXILCONTPOSTPROCESS %s +; RUN: count 0 < %t1.stderr + +target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" + +%dx.types.Handle = type { i8* } +%dx.types.ResourceProperties = type { i32, i32 } +; Doesn't need to spill: +%struct.SmallPayload = type { [1 x i32] } +; These two need to spill: +%struct.MediumPayload = type { [3 x i32] } +%struct.LargePayload = type { [5 x i32] } +%struct.DispatchSystemData = type { <3 x i32> } +%struct.TraversalData = type { %struct.SystemData, %struct.HitData, <3 x float>, <3 x float>, float, i64 } +%struct.SystemData = type { %struct.DispatchSystemData } +%struct.HitData = type { <3 x float>, <3 x float>, float, i32 } +%struct.AnyHitTraversalData = type { %struct.TraversalData, %struct.HitData } +%struct.BuiltInTriangleIntersectionAttributes = type { <2 x float> } + +@"\01?Scene@@3URaytracingAccelerationStructure@@A" = external constant %dx.types.Handle, align 4 + +declare %dx.types.Handle @dx.op.annotateHandle(i32, %dx.types.Handle, %dx.types.ResourceProperties) #3 +declare %dx.types.Handle @dx.op.createHandleForLib.dx.types.Handle(i32, %dx.types.Handle) #4 +declare !types !200 void @dx.op.traceRay.struct.SmallPayload(i32, %dx.types.Handle, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, %struct.SmallPayload*) +declare !types !201 void @dx.op.traceRay.struct.MediumPayload(i32, %dx.types.Handle, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, %struct.MediumPayload*) +declare !types !202 void @dx.op.traceRay.struct.LargePayload(i32, %dx.types.Handle, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, %struct.LargePayload*) + +define void @_cont_TraceRay(%struct.DispatchSystemData* %data, i64 %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, float %6, float %7, float %8, float %9, float %10, float %11, float %12, float %13) #1 !types !203 { + %dis_data = load %struct.DispatchSystemData, %struct.DispatchSystemData* %data, align 4 + %sys_data = insertvalue %struct.SystemData undef, %struct.DispatchSystemData %dis_data, 0 + %trav_data = insertvalue %struct.TraversalData undef, %struct.SystemData %sys_data, 0 + %trav_data2 = insertvalue %struct.TraversalData %trav_data, i64 -1, 5 + %newdata = call %struct.DispatchSystemData @_AmdAwaitTraversal(i64 4, %struct.TraversalData %trav_data2) + store %struct.DispatchSystemData %newdata, %struct.DispatchSystemData* %data, align 4 + call void @_AmdRestoreSystemData(%struct.DispatchSystemData* %data) + ret void +} + +define void @Miss(%struct.SmallPayload* noalias nocapture %outerpayload) !types !204 !continuation.maxHitAttributeBytes !32 { + %p1 = alloca %struct.SmallPayload + %p2 = alloca %struct.MediumPayload + %p3 = alloca %struct.LargePayload + ; Avoid undefs being written to payload registers + ; caused by uninitialized payloads. + store %struct.SmallPayload zeroinitializer, %struct.SmallPayload* %p1 + store %struct.MediumPayload zeroinitializer, %struct.LargePayload* %p2 + store %struct.LargePayload zeroinitializer, %struct.MediumPayload* %p3 + + %t1 = load %dx.types.Handle, %dx.types.Handle* @"\01?Scene@@3URaytracingAccelerationStructure@@A", align 4 + %t2 = call %dx.types.Handle @dx.op.createHandleForLib.dx.types.Handle(i32 160, %dx.types.Handle %t1) + %t3 = call %dx.types.Handle @dx.op.annotateHandle(i32 216, %dx.types.Handle %t2, %dx.types.ResourceProperties { i32 16, i32 0 }) + + call void @dx.op.traceRay.struct.SmallPayload(i32 157, %dx.types.Handle %t3, i32 16, i32 -1, i32 0, i32 1, i32 0, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0x3F50624DE0000000, float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 1.000000e+04, %struct.SmallPayload* nonnull %p1) + call void @dx.op.traceRay.struct.MediumPayload(i32 157, %dx.types.Handle %t3, i32 16, i32 -1, i32 0, i32 1, i32 0, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0x3F50624DE0000000, float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 1.000000e+04, %struct.MediumPayload* nonnull %p2) + call void @dx.op.traceRay.struct.LargePayload(i32 157, %dx.types.Handle %t3, i32 16, i32 -1, i32 0, i32 1, i32 0, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0x3F50624DE0000000, float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 1.000000e+04, %struct.LargePayload* nonnull %p3) + ret void +} + +; Function Attrs: alwaysinline +declare %struct.DispatchSystemData @_cont_SetupRayGen() #1 + +; Function Attrs: alwaysinline +declare %struct.DispatchSystemData @_AmdAwaitTraversal(i64, %struct.TraversalData) #1 + +; Function Attrs: alwaysinline +declare %struct.DispatchSystemData @_AmdAwaitShader(i64, %struct.DispatchSystemData) #1 + +; Function Attrs: alwaysinline +declare %struct.AnyHitTraversalData @_AmdAwaitAnyHit(i64, %struct.AnyHitTraversalData, float, i32) #1 + +; Function Attrs: alwaysinline +declare !types !19 %struct.BuiltInTriangleIntersectionAttributes @_cont_GetTriangleHitAttributes(%struct.SystemData*) #1 + +; Function Attrs: alwaysinline +declare !types !21 void @_cont_SetTriangleHitAttributes(%struct.SystemData*, %struct.BuiltInTriangleIntersectionAttributes) #1 + +; Function Attrs: alwaysinline +declare !types !22 i1 @_cont_IsEndSearch(%struct.TraversalData*) #1 + +; Function Attrs: nounwind memory(read) +declare !types !24 i32 @_cont_HitKind(%struct.SystemData* nocapture readnone, %struct.HitData*) #2 + +; Function Attrs: nounwind memory(none) +declare !types !26 void @_AmdRestoreSystemData(%struct.DispatchSystemData*) #3 + +; Function Attrs: nounwind memory(none) +declare !types !28 void @_AmdRestoreSystemDataAnyHit(%struct.AnyHitTraversalData*) #3 + +; Function Attrs: alwaysinline +define i32 @_cont_GetLocalRootIndex(%struct.DispatchSystemData* %data) #1 !types !30 { + ret i32 5 +} + +attributes #0 = { nounwind } +attributes #1 = { alwaysinline } +attributes #2 = { nounwind memory(read) } +attributes #3 = { nounwind memory(none) } + +!llvm.ident = !{!0} +!dx.version = !{!1} +!dx.valver = !{!1} +!dx.shaderModel = !{!2} +!dx.typeAnnotations = !{!3} +!dx.entryPoints = !{!12, !14} +!continuation.maxPayloadRegisterCount = !{!31} + +!0 = !{!"dxcoob 2019.05.00"} +!1 = !{i32 1, i32 7} +!2 = !{!"lib", i32 6, i32 7} +!3 = !{i32 1, void (%struct.SmallPayload*)* @Miss, !4} +!4 = !{!5, !7} +!5 = !{i32 1, !6, !6} +!6 = !{} +!7 = !{i32 2, !6, !6} +!9 = !{!10, !11, !11} +!10 = !{i32 0, i32 259} +!11 = !{i32 0, i32 513} +!12 = !{null, !"", null, null, !13} +!13 = !{i32 0, i64 32} +!14 = !{void (%struct.SmallPayload*)* @Miss, !"Miss", null, null, !15} +!15 = !{i32 8, i32 11, i32 6, i32 24, i32 5, !16} +!16 = !{i32 0} +!19 = !{!"function", %struct.BuiltInTriangleIntersectionAttributes poison, !20} +!20 = !{i32 0, %struct.SystemData poison} +!21 = !{!"function", !"void", !20, %struct.BuiltInTriangleIntersectionAttributes poison} +!22 = !{!"function", i1 poison, !23} +!23 = !{i32 0, %struct.TraversalData poison} +!24 = !{!"function", i32 poison, !20, !25} +!25 = !{i32 0, %struct.HitData poison} +!26 = !{!"function", !"void", !27} +!27 = !{i32 0, %struct.DispatchSystemData poison} +!28 = !{!"function", !"void", !29} +!29 = !{i32 0, %struct.AnyHitTraversalData poison} +!30 = !{!"function", i32 poison, !27} +!31 = !{i32 2} +!32 = !{i32 8} + +!100 = !{i32 0, %struct.SmallPayload poison} +!101 = !{i32 0, %struct.MediumPayload poison} +!102 = !{i32 0, %struct.LargePayload poison} +!103 = !{i32 0, %struct.DispatchSystemData poison} +!200 = !{!"function", !"void", i32 poison, %dx.types.Handle poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, !100} +!201 = !{!"function", !"void", i32 poison, %dx.types.Handle poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, !101} +!202 = !{!"function", !"void", i32 poison, %dx.types.Handle poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, !102} +!203 = !{!"function", !"void", !103, i64 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison} +!204 = !{!"function", !"void", !100} +; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.DispatchSystemData @Miss( +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) !continuation.maxHitAttributeBytes !16 !lgc.rt.shaderstage [[META17:![0-9]+]] !continuation.registercount [[META18:![0-9]+]] !continuation [[META19:![0-9]+]] !continuation.stacksize [[META20:![0-9]+]] { +; LOWERRAYTRACINGPIPELINE-NEXT: [[P1:%.*]] = alloca [[STRUCT_SMALLPAYLOAD:%.*]], align 8 +; LOWERRAYTRACINGPIPELINE-NEXT: [[P2:%.*]] = alloca [[STRUCT_MEDIUMPAYLOAD:%.*]], align 8 +; LOWERRAYTRACINGPIPELINE-NEXT: [[P3:%.*]] = alloca [[STRUCT_LARGEPAYLOAD:%.*]], align 8 +; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_SMALLPAYLOAD]], align 8 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = call [[STRUCT_SYSTEMDATA]] @continuations.getSystemData.s_struct.SystemDatas() +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_SYSTEMDATA]] [[TMP3]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = call ptr @continuation.getContinuationStackOffset() +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = add i32 [[TMP6]], 16 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP7]], ptr [[TMP5]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_SMALLPAYLOAD]], ptr [[TMP2]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = load [1 x i32], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [1 x i32] [[TMP9]], ptr [[TMP8]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_SMALLPAYLOAD]] zeroinitializer, ptr [[P1]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_MEDIUMPAYLOAD]] zeroinitializer, ptr [[P2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_LARGEPAYLOAD]] zeroinitializer, ptr [[P3]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[T1:%.*]] = load [[DX_TYPES_HANDLE:%.*]], ptr @"\01?Scene@@3URaytracingAccelerationStructure@@A", align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[T2:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[T1]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[T3:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[T2]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[T3]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA_I:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP11]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA_I:%.*]] = insertvalue [[STRUCT_SYSTEMDATA]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I]], 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA_I:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA_I]], 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA2_I:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]], i64 -1, 5 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_SMALLPAYLOAD]], ptr [[P1]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = load [1 x i32], ptr [[TMP12]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [1 x i32] [[TMP13]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = call ptr inttoptr (i64 4 to ptr)([[STRUCT_TRAVERSALDATA]] [[TRAV_DATA2_I]]), !continuation.registercount [[META18]], !continuation.returnedRegistercount !18 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @await.struct.DispatchSystemData(ptr [[TMP14]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_SMALLPAYLOAD]] poison, ptr [[P1]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = getelementptr inbounds [[STRUCT_SMALLPAYLOAD]], ptr [[P1]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = load [1 x i32], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [1 x i32] [[TMP17]], ptr [[TMP16]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP15]], ptr [[TMP11]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; LOWERRAYTRACINGPIPELINE-NEXT: br label [[DOTSPLIT10:%.*]] +; LOWERRAYTRACINGPIPELINE: .split10: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[T3]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP19:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA_I1:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[TMP19]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA_I2:%.*]] = insertvalue [[STRUCT_SYSTEMDATA]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I1]], 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA_I3:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA_I2]], 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA2_I4:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I3]], i64 -1, 5 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP20:%.*]] = call ptr @continuation.getContinuationStackOffset() +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP22:%.*]] = add i32 [[TMP21]], -16 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP22]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP23:%.*]] = getelementptr inbounds [[STRUCT_MEDIUMPAYLOAD]], ptr [[P2]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP24:%.*]] = getelementptr i32, ptr [[TMP23]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr [[TMP24]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP25]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP26]], ptr getelementptr inbounds ([[STRUCT_MEDIUMPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_0_CALLER_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 1), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP27:%.*]] = getelementptr i32, ptr [[TMP24]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = load i32, ptr [[TMP27]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP28]], ptr getelementptr ([[STRUCT_MEDIUMPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 2), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = getelementptr i32, ptr [[TMP24]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP30:%.*]] = load i32, ptr [[TMP29]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP30]], ptr getelementptr ([[STRUCT_MEDIUMPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 3), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP31:%.*]] = call ptr inttoptr (i64 4 to ptr)([[STRUCT_TRAVERSALDATA]] [[TRAV_DATA2_I4]]), !continuation.registercount [[META14:![0-9]+]], !continuation.returnedRegistercount !14 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP32:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @await.struct.DispatchSystemData(ptr [[TMP31]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_MEDIUMPAYLOAD]] poison, ptr [[P2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP33:%.*]] = getelementptr inbounds [[STRUCT_MEDIUMPAYLOAD]], ptr [[P2]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = getelementptr i32, ptr [[TMP33]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP35:%.*]] = getelementptr i32, ptr [[TMP34]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP36:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_MEDIUMPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 1), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP36]], ptr [[TMP35]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP37:%.*]] = getelementptr i32, ptr [[TMP34]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = load i32, ptr getelementptr ([[STRUCT_MEDIUMPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 2), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP38]], ptr [[TMP37]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = getelementptr i32, ptr [[TMP34]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP40:%.*]] = load i32, ptr getelementptr ([[STRUCT_MEDIUMPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 3), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP40]], ptr [[TMP39]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP32]], ptr [[TMP19]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; LOWERRAYTRACINGPIPELINE-NEXT: br label [[DOTSPLIT9:%.*]] +; LOWERRAYTRACINGPIPELINE: .split9: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP41:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[T3]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP42:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA_I5:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[TMP42]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA_I6:%.*]] = insertvalue [[STRUCT_SYSTEMDATA]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I5]], 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA_I7:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA_I6]], 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA2_I8:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I7]], i64 -1, 5 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP43:%.*]] = call ptr @continuation.getContinuationStackOffset() +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP44:%.*]] = load i32, ptr [[TMP43]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP45:%.*]] = add i32 [[TMP44]], -16 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP45]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP46:%.*]] = getelementptr inbounds [[STRUCT_LARGEPAYLOAD]], ptr [[P3]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP47:%.*]] = getelementptr i32, ptr [[TMP46]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP48:%.*]] = getelementptr i32, ptr [[TMP47]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP49:%.*]] = load i32, ptr [[TMP48]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP49]], ptr getelementptr inbounds ([[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_0_CALLER_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 1), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP50:%.*]] = getelementptr i32, ptr [[TMP47]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP51:%.*]] = load i32, ptr [[TMP50]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP51]], ptr getelementptr ([[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 2), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP52:%.*]] = getelementptr i32, ptr [[TMP47]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP53:%.*]] = load i32, ptr [[TMP52]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP53]], ptr getelementptr ([[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 3), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP54:%.*]] = getelementptr i32, ptr [[TMP47]], i64 3 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP55:%.*]] = load i32, ptr [[TMP54]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP55]], ptr getelementptr ([[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 4), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP56:%.*]] = getelementptr i32, ptr [[TMP47]], i64 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP57:%.*]] = load i32, ptr [[TMP56]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP57]], ptr getelementptr ([[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 5), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP58:%.*]] = call ptr inttoptr (i64 4 to ptr)([[STRUCT_TRAVERSALDATA]] [[TRAV_DATA2_I8]]), !continuation.registercount [[META14]], !continuation.returnedRegistercount !14 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP59:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @await.struct.DispatchSystemData(ptr [[TMP58]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_LARGEPAYLOAD]] poison, ptr [[P3]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP60:%.*]] = getelementptr inbounds [[STRUCT_LARGEPAYLOAD]], ptr [[P3]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP61:%.*]] = getelementptr i32, ptr [[TMP60]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP62:%.*]] = getelementptr i32, ptr [[TMP61]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP63:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 1), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP63]], ptr [[TMP62]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP64:%.*]] = getelementptr i32, ptr [[TMP61]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP65:%.*]] = load i32, ptr getelementptr ([[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 2), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP65]], ptr [[TMP64]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP66:%.*]] = getelementptr i32, ptr [[TMP61]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP67:%.*]] = load i32, ptr getelementptr ([[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 3), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP67]], ptr [[TMP66]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP68:%.*]] = getelementptr i32, ptr [[TMP61]], i64 3 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP69:%.*]] = load i32, ptr getelementptr ([[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 4), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP69]], ptr [[TMP68]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP70:%.*]] = getelementptr i32, ptr [[TMP61]], i64 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP71:%.*]] = load i32, ptr getelementptr ([[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 5), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP71]], ptr [[TMP70]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP59]], ptr [[TMP42]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; LOWERRAYTRACINGPIPELINE-NEXT: br label [[DOTSPLIT:%.*]] +; LOWERRAYTRACINGPIPELINE: .split: +; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP72:%.*]] = getelementptr inbounds [[STRUCT_SMALLPAYLOAD]], ptr [[TMP2]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP73:%.*]] = load [1 x i32], ptr [[TMP72]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [1 x i32] [[TMP73]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP74:%.*]] = call ptr @continuation.getContinuationStackOffset() +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP75:%.*]] = load i32, ptr [[TMP74]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP76:%.*]] = add i32 [[TMP75]], -16 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP76]], ptr [[TMP74]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP77:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP78:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[TMP77]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP78]], !continuation.registercount [[META18]] +; +; +; LOWERRAYTRACINGPIPELINE-LABEL: define i32 @_cont_GetLocalRootIndex( +; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]]) #[[ATTR1:[0-9]+]] { +; LOWERRAYTRACINGPIPELINE-NEXT: ret i32 5 +; +; +; DXILCONTPOSTPROCESS-LABEL: define void @Miss( +; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) !continuation.maxHitAttributeBytes !15 !lgc.rt.shaderstage [[META16:![0-9]+]] !continuation.registercount [[META17:![0-9]+]] !continuation [[META18:![0-9]+]] !continuation.stacksize [[META19:![0-9]+]] !continuation.state [[META20:![0-9]+]] { +; DXILCONTPOSTPROCESS-NEXT: AllocaSpillBB: +; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 +; DXILCONTPOSTPROCESS-NEXT: [[CONT_STATE:%.*]] = alloca [3 x i32], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[CSP:%.*]] = alloca i32, align 4 +; DXILCONTPOSTPROCESS-NEXT: store [[STRUCT_SYSTEMDATA]] [[TMP0]], ptr [[SYSTEM_DATA]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[CSPINIT]], ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_SPILL_ADDR:%.*]] = getelementptr inbounds [[MISS_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: store i64 [[RETURNADDR]], ptr [[RETURNADDR_SPILL_ADDR]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP1:%.*]] = load [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_EXTRACT:%.*]] = extractvalue [[STRUCT_SYSTEMDATA]] [[TMP1]], 0, 0 +; DXILCONTPOSTPROCESS-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; DXILCONTPOSTPROCESS-NEXT: [[TMP2:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP3:%.*]] = add i32 [[TMP2]], 16 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP3]], ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [1 x i32] poison, i32 [[TMP4]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_EXTRACT:%.*]] = extractvalue [1 x i32] [[DOTFCA_0_INSERT]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_EXTRACT_SPILL_ADDR:%.*]] = getelementptr inbounds [[MISS_FRAME]], ptr [[CONT_STATE]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[DOTFCA_0_EXTRACT]], ptr [[DOTFCA_0_EXTRACT_SPILL_ADDR]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[T1:%.*]] = load [[DX_TYPES_HANDLE:%.*]], ptr @"\01?Scene@@3URaytracingAccelerationStructure@@A", align 4 +; DXILCONTPOSTPROCESS-NEXT: [[T2:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[T1]]) +; DXILCONTPOSTPROCESS-NEXT: [[T3:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[T2]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) +; DXILCONTPOSTPROCESS-NEXT: [[TMP5:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[T3]]) +; DXILCONTPOSTPROCESS-NEXT: [[DIS_DATA_I_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA:%.*]] poison, <3 x i32> [[DOTFCA_0_0_EXTRACT]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[SYS_DATA_I:%.*]] = insertvalue [[STRUCT_SYSTEMDATA]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I_FCA_0_INSERT]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[TRAV_DATA_I:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA_I]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[TRAV_DATA2_I:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]], i64 -1, 5 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT30:%.*]] = insertvalue [1 x i32] poison, i32 0, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT30_FCA_0_EXTRACT:%.*]] = extractvalue [1 x i32] [[DOTFCA_0_INSERT30]], 0 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[DOTFCA_0_INSERT30_FCA_0_EXTRACT]], ptr addrspace(20) @REGISTERS, align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP6:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP7:%.*]] = add i32 [[TMP6]], 12 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP7]], ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP8:%.*]] = inttoptr i32 [[TMP6]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP9:%.*]] = getelementptr inbounds [3 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP10:%.*]] = getelementptr inbounds [3 x i32], ptr addrspace(21) [[TMP8]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP9]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP11]], ptr addrspace(21) [[TMP10]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP12:%.*]] = getelementptr inbounds [3 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP13:%.*]] = getelementptr inbounds [3 x i32], ptr addrspace(21) [[TMP8]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP14:%.*]] = load i32, ptr [[TMP12]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP14]], ptr addrspace(21) [[TMP13]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP15:%.*]] = getelementptr inbounds [3 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP16:%.*]] = getelementptr inbounds [3 x i32], ptr addrspace(21) [[TMP8]], i32 0, i32 2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP15]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP17]], ptr addrspace(21) [[TMP16]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP18:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP19:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @Miss.resume.0 to i64)) +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 4, i32 [[TMP18]], i64 [[TMP19]], [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA2_I]]), !continuation.registercount [[META17]], !continuation.returnedRegistercount !17 +; DXILCONTPOSTPROCESS-NEXT: unreachable +; +; +; DXILCONTPOSTPROCESS-LABEL: define dso_local void @Miss.resume.0( +; DXILCONTPOSTPROCESS-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !continuation.maxHitAttributeBytes !15 !lgc.rt.shaderstage [[META16]] !continuation.registercount [[META17]] !continuation [[META18]] { +; DXILCONTPOSTPROCESS-NEXT: entryresume.0: +; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA]], align 8 +; DXILCONTPOSTPROCESS-NEXT: [[CONT_STATE:%.*]] = alloca [3 x i32], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[CSP:%.*]] = alloca i32, align 4 +; DXILCONTPOSTPROCESS-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], ptr [[SYSTEM_DATA]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP0]], ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP2:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP3]], i64 -12 +; DXILCONTPOSTPROCESS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [3 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP6:%.*]] = getelementptr inbounds [3 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(21) [[TMP5]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP7]], ptr [[TMP6]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP8:%.*]] = getelementptr inbounds [3 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP9:%.*]] = getelementptr inbounds [3 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP10:%.*]] = load i32, ptr addrspace(21) [[TMP8]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP10]], ptr [[TMP9]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP11:%.*]] = getelementptr inbounds [3 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP12:%.*]] = getelementptr inbounds [3 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP13:%.*]] = load i32, ptr addrspace(21) [[TMP11]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP13]], ptr [[TMP12]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP14:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP15:%.*]] = add i32 [[TMP14]], -12 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP15]], ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP16:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [1 x i32] poison, i32 [[TMP16]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_EXTRACT32:%.*]] = extractvalue [1 x i32] [[DOTFCA_0_INSERT]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_EXTRACT13:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], 0 +; DXILCONTPOSTPROCESS-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; DXILCONTPOSTPROCESS-NEXT: [[T110:%.*]] = load [[DX_TYPES_HANDLE:%.*]], ptr @"\01?Scene@@3URaytracingAccelerationStructure@@A", align 4 +; DXILCONTPOSTPROCESS-NEXT: [[T29:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[T110]]) +; DXILCONTPOSTPROCESS-NEXT: [[T38:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[T29]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) +; DXILCONTPOSTPROCESS-NEXT: [[TMP17:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[T38]]) +; DXILCONTPOSTPROCESS-NEXT: [[DIS_DATA_I1_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, <3 x i32> [[DOTFCA_0_EXTRACT13]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[SYS_DATA_I2:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I1_FCA_0_INSERT]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[TRAV_DATA_I3:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA_I2]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[TRAV_DATA2_I4:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I3]], i64 -1, 5 +; DXILCONTPOSTPROCESS-NEXT: [[TMP18:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP19:%.*]] = add i32 [[TMP18]], -16 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP19]], ptr addrspace(20) @REGISTERS, align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 0, ptr addrspace(20) addrspacecast (ptr getelementptr inbounds ([[STRUCT_MEDIUMPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_0_CALLER_OUT:%.*]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i32 1) to ptr addrspace(20)), align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP20:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP21:%.*]] = inttoptr i32 [[TMP20]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP21]], i32 -2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP23:%.*]] = getelementptr [[STRUCT_MEDIUMPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP22]], i32 0, i32 0, i64 2 +; DXILCONTPOSTPROCESS-NEXT: store i32 0, ptr addrspace(21) [[TMP23]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP24:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP25:%.*]] = inttoptr i32 [[TMP24]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP26:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP25]], i32 -2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP27:%.*]] = getelementptr [[STRUCT_MEDIUMPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP26]], i32 0, i32 0, i64 3 +; DXILCONTPOSTPROCESS-NEXT: store i32 0, ptr addrspace(21) [[TMP27]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP28:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP29:%.*]] = add i32 [[TMP28]], 12 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP29]], ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP30:%.*]] = inttoptr i32 [[TMP28]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP31:%.*]] = getelementptr inbounds [3 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP32:%.*]] = getelementptr inbounds [3 x i32], ptr addrspace(21) [[TMP30]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP33:%.*]] = load i32, ptr [[TMP31]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP33]], ptr addrspace(21) [[TMP32]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP34:%.*]] = getelementptr inbounds [3 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP35:%.*]] = getelementptr inbounds [3 x i32], ptr addrspace(21) [[TMP30]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP36:%.*]] = load i32, ptr [[TMP34]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP36]], ptr addrspace(21) [[TMP35]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP37:%.*]] = getelementptr inbounds [3 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP38:%.*]] = getelementptr inbounds [3 x i32], ptr addrspace(21) [[TMP30]], i32 0, i32 2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP39:%.*]] = load i32, ptr [[TMP37]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP39]], ptr addrspace(21) [[TMP38]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP40:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP41:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @Miss.resume.1 to i64)) +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 4, i32 [[TMP40]], i64 [[TMP41]], [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA2_I4]]), !continuation.registercount [[META13:![0-9]+]], !continuation.returnedRegistercount !13 +; DXILCONTPOSTPROCESS-NEXT: unreachable +; +; +; DXILCONTPOSTPROCESS-LABEL: define dso_local void @Miss.resume.1( +; DXILCONTPOSTPROCESS-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !continuation.maxHitAttributeBytes !15 !lgc.rt.shaderstage [[META16]] !continuation.registercount [[META13]] !continuation [[META18]] { +; DXILCONTPOSTPROCESS-NEXT: entryresume.1: +; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA]], align 8 +; DXILCONTPOSTPROCESS-NEXT: [[CONT_STATE:%.*]] = alloca [3 x i32], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[CSP:%.*]] = alloca i32, align 4 +; DXILCONTPOSTPROCESS-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], ptr [[SYSTEM_DATA]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP0]], ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP2:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP3]], i64 -12 +; DXILCONTPOSTPROCESS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [3 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP6:%.*]] = getelementptr inbounds [3 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(21) [[TMP5]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP7]], ptr [[TMP6]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP8:%.*]] = getelementptr inbounds [3 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP9:%.*]] = getelementptr inbounds [3 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP10:%.*]] = load i32, ptr addrspace(21) [[TMP8]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP10]], ptr [[TMP9]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP11:%.*]] = getelementptr inbounds [3 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP12:%.*]] = getelementptr inbounds [3 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP13:%.*]] = load i32, ptr addrspace(21) [[TMP11]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP13]], ptr [[TMP12]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP14:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP15:%.*]] = add i32 [[TMP14]], -12 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP15]], ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP16:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr inbounds ([[STRUCT_MEDIUMPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i32 1) to ptr addrspace(20)), align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP17:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP18:%.*]] = inttoptr i32 [[TMP17]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP18]], i32 -2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP20:%.*]] = getelementptr [[STRUCT_MEDIUMPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP19]], i32 0, i32 0, i64 2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP21:%.*]] = load i32, ptr addrspace(21) [[TMP20]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP22:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP23:%.*]] = inttoptr i32 [[TMP22]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP24:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP23]], i32 -2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP25:%.*]] = getelementptr [[STRUCT_MEDIUMPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP24]], i32 0, i32 0, i64 3 +; DXILCONTPOSTPROCESS-NEXT: [[TMP26:%.*]] = load i32, ptr addrspace(21) [[TMP25]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_EXTRACT15:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], 0 +; DXILCONTPOSTPROCESS-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; DXILCONTPOSTPROCESS-NEXT: [[T17:%.*]] = load [[DX_TYPES_HANDLE:%.*]], ptr @"\01?Scene@@3URaytracingAccelerationStructure@@A", align 4 +; DXILCONTPOSTPROCESS-NEXT: [[T26:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[T17]]) +; DXILCONTPOSTPROCESS-NEXT: [[T35:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[T26]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) +; DXILCONTPOSTPROCESS-NEXT: [[TMP27:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[T35]]) +; DXILCONTPOSTPROCESS-NEXT: [[DIS_DATA_I5_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, <3 x i32> [[DOTFCA_0_EXTRACT15]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[SYS_DATA_I6:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I5_FCA_0_INSERT]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[TRAV_DATA_I7:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA_I6]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[TRAV_DATA2_I8:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I7]], i64 -1, 5 +; DXILCONTPOSTPROCESS-NEXT: [[TMP28:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP29:%.*]] = add i32 [[TMP28]], -16 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP29]], ptr addrspace(20) @REGISTERS, align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 0, ptr addrspace(20) addrspacecast (ptr getelementptr inbounds ([[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_0_CALLER_OUT:%.*]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i32 1) to ptr addrspace(20)), align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP30:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP31:%.*]] = inttoptr i32 [[TMP30]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP31]], i32 -2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP33:%.*]] = getelementptr [[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP32]], i32 0, i32 0, i64 2 +; DXILCONTPOSTPROCESS-NEXT: store i32 0, ptr addrspace(21) [[TMP33]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP34:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP35:%.*]] = inttoptr i32 [[TMP34]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP36:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP35]], i32 -2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP37:%.*]] = getelementptr [[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP36]], i32 0, i32 0, i64 3 +; DXILCONTPOSTPROCESS-NEXT: store i32 0, ptr addrspace(21) [[TMP37]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP38:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP39:%.*]] = inttoptr i32 [[TMP38]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP40:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP39]], i32 -2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP41:%.*]] = getelementptr [[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP40]], i32 0, i32 0, i64 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 0, ptr addrspace(21) [[TMP41]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP42:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP43:%.*]] = inttoptr i32 [[TMP42]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP44:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP43]], i32 -2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP45:%.*]] = getelementptr [[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP44]], i32 0, i32 0, i64 5 +; DXILCONTPOSTPROCESS-NEXT: store i32 0, ptr addrspace(21) [[TMP45]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP46:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP47:%.*]] = add i32 [[TMP46]], 12 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP47]], ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP48:%.*]] = inttoptr i32 [[TMP46]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP49:%.*]] = getelementptr inbounds [3 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP50:%.*]] = getelementptr inbounds [3 x i32], ptr addrspace(21) [[TMP48]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP51:%.*]] = load i32, ptr [[TMP49]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP51]], ptr addrspace(21) [[TMP50]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP52:%.*]] = getelementptr inbounds [3 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP53:%.*]] = getelementptr inbounds [3 x i32], ptr addrspace(21) [[TMP48]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP54:%.*]] = load i32, ptr [[TMP52]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP54]], ptr addrspace(21) [[TMP53]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP55:%.*]] = getelementptr inbounds [3 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP56:%.*]] = getelementptr inbounds [3 x i32], ptr addrspace(21) [[TMP48]], i32 0, i32 2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP57:%.*]] = load i32, ptr [[TMP55]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP57]], ptr addrspace(21) [[TMP56]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP58:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP59:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @Miss.resume.2 to i64)) +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 4, i32 [[TMP58]], i64 [[TMP59]], [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA2_I8]]), !continuation.registercount [[META13]], !continuation.returnedRegistercount !13 +; DXILCONTPOSTPROCESS-NEXT: unreachable +; +; +; DXILCONTPOSTPROCESS-LABEL: define dso_local void @Miss.resume.2( +; DXILCONTPOSTPROCESS-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !continuation.maxHitAttributeBytes !15 !lgc.rt.shaderstage [[META16]] !continuation.registercount [[META13]] !continuation [[META18]] { +; DXILCONTPOSTPROCESS-NEXT: entryresume.2: +; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA]], align 8 +; DXILCONTPOSTPROCESS-NEXT: [[CONT_STATE:%.*]] = alloca [3 x i32], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[CSP:%.*]] = alloca i32, align 4 +; DXILCONTPOSTPROCESS-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], ptr [[SYSTEM_DATA]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP0]], ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP2:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP3]], i64 -12 +; DXILCONTPOSTPROCESS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [3 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP6:%.*]] = getelementptr inbounds [3 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(21) [[TMP5]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP7]], ptr [[TMP6]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP8:%.*]] = getelementptr inbounds [3 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP9:%.*]] = getelementptr inbounds [3 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP10:%.*]] = load i32, ptr addrspace(21) [[TMP8]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP10]], ptr [[TMP9]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP11:%.*]] = getelementptr inbounds [3 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP12:%.*]] = getelementptr inbounds [3 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP13:%.*]] = load i32, ptr addrspace(21) [[TMP11]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP13]], ptr [[TMP12]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP14:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP15:%.*]] = add i32 [[TMP14]], -12 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP15]], ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP16:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr inbounds ([[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i32 1) to ptr addrspace(20)), align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP17:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP18:%.*]] = inttoptr i32 [[TMP17]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP18]], i32 -2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP20:%.*]] = getelementptr [[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP19]], i32 0, i32 0, i64 2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP21:%.*]] = load i32, ptr addrspace(21) [[TMP20]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP22:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP23:%.*]] = inttoptr i32 [[TMP22]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP24:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP23]], i32 -2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP25:%.*]] = getelementptr [[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP24]], i32 0, i32 0, i64 3 +; DXILCONTPOSTPROCESS-NEXT: [[TMP26:%.*]] = load i32, ptr addrspace(21) [[TMP25]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP27:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP28:%.*]] = inttoptr i32 [[TMP27]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP29:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP28]], i32 -2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP30:%.*]] = getelementptr [[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP29]], i32 0, i32 0, i64 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP31:%.*]] = load i32, ptr addrspace(21) [[TMP30]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP32:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP33:%.*]] = inttoptr i32 [[TMP32]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP34:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP33]], i32 -2 +; DXILCONTPOSTPROCESS-NEXT: [[TMP35:%.*]] = getelementptr [[STRUCT_LARGEPAYLOAD_ATTR_MAX_2_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP34]], i32 0, i32 0, i64 5 +; DXILCONTPOSTPROCESS-NEXT: [[TMP36:%.*]] = load i32, ptr addrspace(21) [[TMP35]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_EXTRACT17:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], 0 +; DXILCONTPOSTPROCESS-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_EXTRACT_RELOAD_ADDR:%.*]] = getelementptr inbounds [[MISS_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_EXTRACT_RELOAD:%.*]] = load i32, ptr [[DOTFCA_0_EXTRACT_RELOAD_ADDR]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[MISS_FRAME]], ptr [[CONT_STATE]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_RELOAD:%.*]] = load i64, ptr [[RETURNADDR_RELOAD_ADDR]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [1 x i32] poison, i32 [[DOTFCA_0_EXTRACT_RELOAD]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT_FCA_0_EXTRACT:%.*]] = extractvalue [1 x i32] [[DOTFCA_0_INSERT]], 0 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[DOTFCA_0_INSERT_FCA_0_EXTRACT]], ptr addrspace(20) @REGISTERS, align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP37:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP38:%.*]] = add i32 [[TMP37]], -16 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP38]], ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT12:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, <3 x i32> [[DOTFCA_0_EXTRACT17]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP39:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP39]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT12]]), !continuation.registercount [[META17]] +; DXILCONTPOSTPROCESS-NEXT: unreachable +; +; +; DXILCONTPOSTPROCESS-LABEL: define i32 @_cont_GetLocalRootIndex( +; DXILCONTPOSTPROCESS-SAME: ptr [[DATA:%.*]]) #[[ATTR1:[0-9]+]] { +; DXILCONTPOSTPROCESS-NEXT: ret i32 5 +; diff --git a/shared/continuations/test/dx/lower-rt-pipeline-multiple-traceray-per-payload.ll b/shared/continuations/test/dx/lower-rt-pipeline-multiple-traceray-per-payload.ll deleted file mode 100644 index d54c8705e5..0000000000 --- a/shared/continuations/test/dx/lower-rt-pipeline-multiple-traceray-per-payload.ll +++ /dev/null @@ -1,342 +0,0 @@ -; Test that multiple copies of TraceRay are generated for the same payload type -; if required by e.g. different shader configs (max hit attr size). -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2>%t.stderr | FileCheck %s -; RUN: count 0 < %t.stderr - -target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" - -%dx.types.Handle = type { i8* } -%struct.DispatchSystemData = type { <3 x i32> } -%struct.TraversalData = type { %struct.SystemData, %struct.HitData, <3 x float>, <3 x float>, float, i64 } -%struct.SystemData = type { %struct.DispatchSystemData } -%struct.HitData = type { <3 x float>, <3 x float>, float, i32 } -%struct.AnyHitTraversalData = type { %struct.TraversalData, %struct.HitData } -%struct.BuiltInTriangleIntersectionAttributes = type { <2 x float> } -%struct.RayPayload = type { <4 x float> } -%dx.types.ResourceProperties = type { i32, i32 } -%struct.RaytracingAccelerationStructure = type { i32 } -%"class.RWTexture2D >" = type { <4 x float> } - -@"\01?Scene@@3URaytracingAccelerationStructure@@A" = external constant %dx.types.Handle, align 4 -@"\01?RenderTarget@@3V?$RWTexture2D@V?$vector@M$03@@@@A" = external constant %dx.types.Handle, align 4 - -declare i32 @_cont_GetContinuationStackAddr() #0 - -declare %struct.DispatchSystemData @_cont_SetupRayGen() #0 - -declare %struct.DispatchSystemData @_AmdAwaitTraversal(i64, %struct.TraversalData) #0 - -declare %struct.DispatchSystemData @_AmdAwaitShader(i64, %struct.DispatchSystemData) #0 - -declare %struct.AnyHitTraversalData @_AmdAwaitAnyHit(i64, %struct.AnyHitTraversalData, float, i32) #0 - -declare !types !16 %struct.HitData @_cont_GetCandidateState(%struct.AnyHitTraversalData*) #0 - -declare !types !18 %struct.HitData @_cont_GetCommittedState(%struct.SystemData*) #0 - -declare !types !20 %struct.BuiltInTriangleIntersectionAttributes @_cont_GetTriangleHitAttributes(%struct.SystemData*) #0 - -declare !types !21 void @_cont_SetTriangleHitAttributes(%struct.SystemData*, %struct.BuiltInTriangleIntersectionAttributes) #0 - -declare !types !22 i32 @_cont_GetLocalRootIndex(%struct.DispatchSystemData*) - -declare !types !24 i1 @_cont_IsEndSearch(%struct.TraversalData*) #0 - -declare !types !26 i32 @_cont_HitKind(%struct.SystemData*) #0 - -; Function Attrs: nounwind -declare i64 @_AmdGetResumePointAddr() #1 - -; Function Attrs: nounwind -declare !types !27 void @_AmdRestoreSystemData(%struct.DispatchSystemData*) #1 - -; Function Attrs: nounwind -declare !types !28 void @_AmdRestoreSystemDataAnyHit(%struct.AnyHitTraversalData*) #1 - -; Function Attrs: nounwind -declare !types !27 void @_cont_AcceptHitAndEndSearch(%struct.DispatchSystemData* nocapture readnone) #1 - -; Function Attrs: nounwind -declare !types !28 void @_cont_AcceptHit(%struct.AnyHitTraversalData* nocapture readnone) #1 - -; Function Attrs: nounwind -declare !types !27 void @_cont_IgnoreHit(%struct.DispatchSystemData* nocapture readnone) #1 - -; Function Attrs: nounwind -declare !types !28 void @_AmdAcceptHitAttributes(%struct.AnyHitTraversalData* nocapture readnone) #1 - -; CHECK: define{{.*}}_cont_TraceRay.struct.RayPayload.attr_max_8_bytes( -; CHECK: define{{.*}}_cont_TraceRay.struct.RayPayload.attr_max_24_bytes( -define void @_cont_TraceRay(%struct.DispatchSystemData* %data, i64 %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, float %6, float %7, float %8, float %9, float %10, float %11, float %12, float %13) !types !29 { - %dis_data = load %struct.DispatchSystemData, %struct.DispatchSystemData* %data, align 4 - %sys_data = insertvalue %struct.SystemData undef, %struct.DispatchSystemData %dis_data, 0 - %trav_data = insertvalue %struct.TraversalData undef, %struct.SystemData %sys_data, 0 - %addr = call i64 @_AmdGetResumePointAddr() #3 - %trav_data2 = insertvalue %struct.TraversalData %trav_data, i64 %addr, 5 - %newdata = call %struct.DispatchSystemData @_AmdAwaitTraversal(i64 4, %struct.TraversalData %trav_data2) - store %struct.DispatchSystemData %newdata, %struct.DispatchSystemData* %data, align 4 - call void @_AmdRestoreSystemData(%struct.DispatchSystemData* %data) - ret void -} - -define void @_cont_CallShader(%struct.DispatchSystemData* %data, i32 %0) #0 !types !30 { - %dis_data = load %struct.DispatchSystemData, %struct.DispatchSystemData* %data, align 4 - %newdata = call %struct.DispatchSystemData @_AmdAwaitShader(i64 2, %struct.DispatchSystemData %dis_data) - store %struct.DispatchSystemData %newdata, %struct.DispatchSystemData* %data, align 4 - call void @_AmdRestoreSystemData(%struct.DispatchSystemData* %data) - ret void -} - -define i1 @_cont_ReportHit(%struct.AnyHitTraversalData* %data, float %t, i32 %hitKind) !types !31 { - %origTPtr = getelementptr inbounds %struct.AnyHitTraversalData, %struct.AnyHitTraversalData* %data, i32 0, i32 0, i32 4 - %origT = load float, float* %origTPtr, align 4 - %isNoHit = fcmp fast uge float %t, %origT - br i1 %isNoHit, label %isEnd, label %callAHit - -callAHit: ; preds = %0 - %trav_data = load %struct.AnyHitTraversalData, %struct.AnyHitTraversalData* %data, align 4 - %newdata = call %struct.AnyHitTraversalData @_AmdAwaitAnyHit(i64 3, %struct.AnyHitTraversalData %trav_data, float %t, i32 %hitKind) - store %struct.AnyHitTraversalData %newdata, %struct.AnyHitTraversalData* %data, align 4 - call void @_AmdRestoreSystemDataAnyHit(%struct.AnyHitTraversalData* %data) - ret i1 true - -isEnd: ; preds = %0 - call void @_AmdAcceptHitAttributes(%struct.AnyHitTraversalData* %data) - ret i1 false -} - -define i32 @_cont_DispatchRaysIndex(%struct.DispatchSystemData* %data, i32 %i) !types !32 { - %resPtr = getelementptr %struct.DispatchSystemData, %struct.DispatchSystemData* %data, i32 0, i32 0, i32 %i - %res = load i32, i32* %resPtr, align 4 - ret i32 %res -} - -define float @_cont_ObjectRayOrigin(%struct.DispatchSystemData* nocapture readnone %data, %struct.HitData* %hitData, i32 %i) !types !33 { - %resPtr = getelementptr %struct.HitData, %struct.HitData* %hitData, i32 0, i32 0, i32 %i - %res = load float, float* %resPtr, align 4 - ret float %res -} - -define float @_cont_ObjectRayDirection(%struct.DispatchSystemData* nocapture readnone %data, %struct.HitData* %hitData, i32 %i) !types !33 { - %resPtr = getelementptr %struct.HitData, %struct.HitData* %hitData, i32 0, i32 1, i32 %i - %res = load float, float* %resPtr, align 4 - ret float %res -} - -define float @_cont_RayTCurrent(%struct.DispatchSystemData* nocapture readnone %data, %struct.HitData* %hitData) !types !35 { - %resPtr = getelementptr %struct.HitData, %struct.HitData* %hitData, i32 0, i32 2 - %res = load float, float* %resPtr, align 4 - ret float %res -} - -define void @MyRayGen8() !continuation.maxHitAttributeBytes !36 { - %1 = load %dx.types.Handle, %dx.types.Handle* @"\01?Scene@@3URaytracingAccelerationStructure@@A", align 4 - %2 = load %dx.types.Handle, %dx.types.Handle* @"\01?RenderTarget@@3V?$RWTexture2D@V?$vector@M$03@@@@A", align 4 - %3 = alloca %struct.RayPayload, align 4 - %4 = bitcast %struct.RayPayload* %3 to i8* - call void @llvm.lifetime.start.p0i8(i64 16, i8* %4) #1 - %5 = getelementptr inbounds %struct.RayPayload, %struct.RayPayload* %3, i32 0, i32 0 - store <4 x float> zeroinitializer, <4 x float>* %5, align 4, !tbaa !37 - %6 = call %dx.types.Handle @dx.op.createHandleForLib.dx.types.Handle(i32 160, %dx.types.Handle %1) - %7 = call %dx.types.Handle @dx.op.annotateHandle(i32 216, %dx.types.Handle %6, %dx.types.ResourceProperties { i32 16, i32 0 }) - call void @dx.op.traceRay.struct.RayPayload(i32 157, %dx.types.Handle %7, i32 16, i32 -1, i32 0, i32 1, i32 0, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0x3F50624DE0000000, float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 1.000000e+04, %struct.RayPayload* nonnull %3) - %8 = load <4 x float>, <4 x float>* %5, align 4, !tbaa !37 - %9 = call i32 @dx.op.dispatchRaysIndex.i32(i32 145, i8 0) - %10 = call i32 @dx.op.dispatchRaysIndex.i32(i32 145, i8 1) - %11 = call %dx.types.Handle @dx.op.createHandleForLib.dx.types.Handle(i32 160, %dx.types.Handle %2) - %12 = call %dx.types.Handle @dx.op.annotateHandle(i32 216, %dx.types.Handle %11, %dx.types.ResourceProperties { i32 4098, i32 1033 }) - %13 = extractelement <4 x float> %8, i64 0 - %14 = extractelement <4 x float> %8, i64 1 - %15 = extractelement <4 x float> %8, i64 2 - %16 = extractelement <4 x float> %8, i64 3 - call void @dx.op.textureStore.f32(i32 67, %dx.types.Handle %12, i32 %9, i32 %10, i32 undef, float %13, float %14, float %15, float %16, i8 15) - call void @llvm.lifetime.end.p0i8(i64 16, i8* %4) #1 - ret void -} - -define void @MyRayGen24() !continuation.maxHitAttributeBytes !40 { - %1 = load %dx.types.Handle, %dx.types.Handle* @"\01?Scene@@3URaytracingAccelerationStructure@@A", align 4 - %2 = load %dx.types.Handle, %dx.types.Handle* @"\01?RenderTarget@@3V?$RWTexture2D@V?$vector@M$03@@@@A", align 4 - %3 = alloca %struct.RayPayload, align 4 - %4 = bitcast %struct.RayPayload* %3 to i8* - call void @llvm.lifetime.start.p0i8(i64 16, i8* %4) #1 - %5 = getelementptr inbounds %struct.RayPayload, %struct.RayPayload* %3, i32 0, i32 0 - store <4 x float> zeroinitializer, <4 x float>* %5, align 4, !tbaa !37 - %6 = call %dx.types.Handle @dx.op.createHandleForLib.dx.types.Handle(i32 160, %dx.types.Handle %1) - %7 = call %dx.types.Handle @dx.op.annotateHandle(i32 216, %dx.types.Handle %6, %dx.types.ResourceProperties { i32 16, i32 0 }) - call void @dx.op.traceRay.struct.RayPayload(i32 157, %dx.types.Handle %7, i32 16, i32 -1, i32 0, i32 1, i32 0, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0x3F50624DE0000000, float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 1.000000e+04, %struct.RayPayload* nonnull %3) - %8 = load <4 x float>, <4 x float>* %5, align 4, !tbaa !37 - %9 = call i32 @dx.op.dispatchRaysIndex.i32(i32 145, i8 0) - %10 = call i32 @dx.op.dispatchRaysIndex.i32(i32 145, i8 1) - %11 = call %dx.types.Handle @dx.op.createHandleForLib.dx.types.Handle(i32 160, %dx.types.Handle %2) - %12 = call %dx.types.Handle @dx.op.annotateHandle(i32 216, %dx.types.Handle %11, %dx.types.ResourceProperties { i32 4098, i32 1033 }) - %13 = extractelement <4 x float> %8, i64 0 - %14 = extractelement <4 x float> %8, i64 1 - %15 = extractelement <4 x float> %8, i64 2 - %16 = extractelement <4 x float> %8, i64 3 - call void @dx.op.textureStore.f32(i32 67, %dx.types.Handle %12, i32 %9, i32 %10, i32 undef, float %13, float %14, float %15, float %16, i8 15) - call void @llvm.lifetime.end.p0i8(i64 16, i8* %4) #1 - ret void -} - -; Function Attrs: nounwind -define void @MyClosestHitShader(%struct.RayPayload* noalias nocapture %payload, %struct.BuiltInTriangleIntersectionAttributes* nocapture readonly %attr) #2 !types !41 { - %1 = getelementptr inbounds %struct.BuiltInTriangleIntersectionAttributes, %struct.BuiltInTriangleIntersectionAttributes* %attr, i32 0, i32 0 - %2 = load <2 x float>, <2 x float>* %1, align 4 - %3 = extractelement <2 x float> %2, i32 0 - %4 = fsub fast float 1.000000e+00, %3 - %5 = extractelement <2 x float> %2, i32 1 - %6 = fsub fast float %4, %5 - %7 = insertelement <4 x float> undef, float %6, i64 0 - %8 = insertelement <4 x float> %7, float %3, i64 1 - %9 = insertelement <4 x float> %8, float %5, i64 2 - %10 = insertelement <4 x float> %9, float 1.000000e+00, i64 3 - %11 = getelementptr inbounds %struct.RayPayload, %struct.RayPayload* %payload, i32 0, i32 0 - store <4 x float> %10, <4 x float>* %11, align 4 - ret void -} - -; Function Attrs: nounwind -define void @MyAnyHitShader(%struct.RayPayload* noalias nocapture %payload, %struct.BuiltInTriangleIntersectionAttributes* nocapture readnone %attr) #2 !types !41 { - %1 = getelementptr inbounds %struct.RayPayload, %struct.RayPayload* %payload, i32 0, i32 0 - %2 = load <4 x float>, <4 x float>* %1, align 4 - %3 = call float @dx.op.objectRayOrigin.f32(i32 149, i8 0) - %4 = call float @dx.op.objectRayDirection.f32(i32 150, i8 0) - %5 = call float @dx.op.rayTCurrent.f32(i32 154) - %6 = fmul fast float %5, %4 - %7 = fadd fast float %6, %3 - %8 = fcmp fast ogt float %7, 0.000000e+00 - %9 = fcmp fast ogt float %7, 1.000000e+00 - %10 = fcmp fast ogt float %7, -1.000000e+00 - br i1 %8, label %11, label %14 - -11: ; preds = %0 - store <4 x float> %2, <4 x float>* %1, align 4 - br i1 %9, label %12, label %13 - -12: ; preds = %11 - call void @dx.op.acceptHitAndEndSearch(i32 156) - unreachable - -13: ; preds = %11 - call void @dx.op.acceptHitAndEndSearch(i32 156) - ret void - -14: ; preds = %0 - br i1 %10, label %15, label %18 - -15: ; preds = %14 - br i1 %9, label %16, label %17 - -16: ; preds = %15 - call void @dx.op.ignoreHit(i32 155) - unreachable - -17: ; preds = %15 - call void @dx.op.ignoreHit(i32 155) - ret void - -18: ; preds = %14 - store <4 x float> %2, <4 x float>* %1, align 4 - ret void -} - -; Function Attrs: nounwind -declare !types !44 void @dx.op.traceRay.struct.RayPayload(i32, %dx.types.Handle, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, %struct.RayPayload*) #1 - -; Function Attrs: nounwind -declare void @dx.op.textureStore.f32(i32, %dx.types.Handle, i32, i32, i32, float, float, float, float, i8) #1 - -; Function Attrs: nounwind memory(none) -declare i32 @dx.op.dispatchRaysIndex.i32(i32, i8) #3 - -; Function Attrs: nounwind memory(none) -declare float @dx.op.objectRayDirection.f32(i32, i8) #3 - -; Function Attrs: nounwind memory(none) -declare float @dx.op.objectRayOrigin.f32(i32, i8) #3 - -; Function Attrs: nounwind memory(read) -declare float @dx.op.rayTCurrent.f32(i32) #4 - -declare void @dx.op.acceptHitAndEndSearch(i32) #0 - -declare void @dx.op.ignoreHit(i32) #0 - -; Function Attrs: nounwind -declare !types !45 i1 @dx.op.reportHit.struct.BuiltInTriangleIntersectionAttributes(i32, float, i32, %struct.BuiltInTriangleIntersectionAttributes*) #1 - -; Function Attrs: nounwind memory(none) -declare %dx.types.Handle @dx.op.annotateHandle(i32, %dx.types.Handle, %dx.types.ResourceProperties) #3 - -; Function Attrs: nounwind memory(read) -declare %dx.types.Handle @dx.op.createHandleForLib.dx.types.Handle(i32, %dx.types.Handle) #4 - -; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) -declare !types !46 void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #5 - -; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) -declare !types !46 void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #5 - -attributes #0 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="0" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #1 = { nounwind } -attributes #2 = { nounwind "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-realign-stack" "stack-protector-buffer-size"="0" "unsafe-fp-math"="false" "use-soft-float"="false" } -attributes #3 = { nounwind memory(none) } -attributes #4 = { nounwind memory(read) } -attributes #5 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } - -!llvm.ident = !{!0} -!dx.version = !{!1} -!dx.valver = !{!1} -!dx.shaderModel = !{!2} -!dx.resources = !{!3} -!dx.entryPoints = !{!10, !12, !15} - -!0 = !{!"clang version 3.7.0 (tags/RELEASE_370/final)"} -!1 = !{i32 1, i32 6} -!2 = !{!"lib", i32 6, i32 6} -!3 = !{!4, !7, null, null} -!4 = !{!5} -!5 = !{i32 0, %struct.RaytracingAccelerationStructure* bitcast (%dx.types.Handle* @"\01?Scene@@3URaytracingAccelerationStructure@@A" to %struct.RaytracingAccelerationStructure*), !"Scene", i32 0, i32 0, i32 1, i32 16, i32 0, !6} -!6 = !{i32 0, i32 4} -!7 = !{!8} -!8 = !{i32 0, %"class.RWTexture2D >"* bitcast (%dx.types.Handle* @"\01?RenderTarget@@3V?$RWTexture2D@V?$vector@M$03@@@@A" to %"class.RWTexture2D >"*), !"RenderTarget", i32 0, i32 0, i32 1, i32 2, i1 false, i1 false, i1 false, !9} -!9 = !{i32 0, i32 9} -!10 = !{null, !"", null, !3, !11} -!11 = !{i32 0, i64 65536} -!12 = !{void ()* @MyRayGen24, !"MyRayGen24", null, null, !13} -!13 = !{i32 8, i32 7, i32 5, !14} -!14 = !{i32 0} -!15 = !{void ()* @MyRayGen8, !"MyRayGen8", null, null, !13} -!16 = !{!"function", %struct.HitData poison, !17} -!17 = !{i32 0, %struct.AnyHitTraversalData poison} -!18 = !{!"function", %struct.HitData poison, !19} -!19 = !{i32 0, %struct.SystemData poison} -!20 = !{!"function", %struct.BuiltInTriangleIntersectionAttributes poison, !19} -!21 = !{!"function", !"void", !19, %struct.BuiltInTriangleIntersectionAttributes poison} -!22 = !{!"function", !"i32", !23} -!23 = !{i32 0, %struct.DispatchSystemData poison} -!24 = !{!"function", i1 poison, !25} -!25 = !{i32 0, %struct.TraversalData poison} -!26 = !{!"function", !"i32", !19} -!27 = !{!"function", !"void", !23} -!28 = !{!"function", !"void", !17} -!29 = !{!"function", !"void", !23, i64 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison} -!30 = !{!"function", !"void", !23, i32 poison} -!31 = !{!"function", !"i1", !17, float poison, i32 poison} -!32 = !{!"function", i32 poison, !23, i32 poison} -!33 = !{!"function", float poison, !23, !34, i32 poison} -!34 = !{i32 0, %struct.HitData poison} -!35 = !{!"function", float poison, !23, !34} -!36 = !{i32 8} -!37 = !{!38, !38, i64 0} -!38 = !{!"omnipotent char", !39, i64 0} -!39 = !{!"Simple C/C++ TBAA"} -!40 = !{i32 24} -!41 = !{!"function", !"void", !42, !43} -!42 = !{i32 0, %struct.RayPayload poison} -!43 = !{i32 0, %struct.BuiltInTriangleIntersectionAttributes poison} -!44 = !{!"function", !"void", i32 poison, %dx.types.Handle poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, !42} -!45 = !{!"function", !"void", i32 poison, float poison, i32 poison, !43} -!46 = !{!"function", !"void", i64 poison, !47} -!47 = !{i32 0, i8 poison} diff --git a/shared/continuations/test/dx/lower-rt-pipeline-simple-call-shader.ll b/shared/continuations/test/dx/lower-rt-pipeline-simple-call-shader.ll index f69cb7323b..6b1f8dfd0b 100644 --- a/shared/continuations/test/dx/lower-rt-pipeline-simple-call-shader.ll +++ b/shared/continuations/test/dx/lower-rt-pipeline-simple-call-shader.ll @@ -1,16 +1,16 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 3 -; RUN: grep -v lgc.cps.module %s | opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S 2>%t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s +; RUN: grep -v lgc.cps.module %s | opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S 2> %t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s ; RUN: count 0 < %t0.stderr -; RUN: grep -v lgc.cps.module %s | opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,remove-types-metadata' \ -; RUN: -S 2>%t1.stderr | FileCheck -check-prefix=CLEANUP %s +; RUN: grep -v lgc.cps.module %s | opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,remove-types-metadata' \ +; RUN: -S 2> %t1.stderr | FileCheck -check-prefix=CLEANUP %s ; RUN: count 0 < %t1.stderr -; RUN: grep -v lgc.cps.module %s | opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,remove-types-metadata' \ -; RUN: -S 2>%t2.stderr | FileCheck -check-prefix=SAVESTATE %s +; RUN: grep -v lgc.cps.module %s | opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,remove-types-metadata' \ +; RUN: -S 2> %t2.stderr | FileCheck -check-prefix=SAVESTATE %s ; RUN: count 0 < %t2.stderr -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2>%t3.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE-CPS %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2> %t3.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE-CPS %s ; RUN: count 0 < %t3.stderr -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,cleanup-continuations,lint,remove-types-metadata' \ -; RUN: -S %s 2>%t4.stderr | FileCheck -check-prefix=CLEANUP-CPS %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,cleanup-continuations,lint,remove-types-metadata' \ +; RUN: -S %s 2> %t4.stderr | FileCheck -check-prefix=CLEANUP-CPS %s ; RUN: count 0 < %t4.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" @@ -94,55 +94,37 @@ attributes #1 = { alwaysinline } ; LOWERRAYTRACINGPIPELINE-NEXT: ret i32 5 ; ; -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_CallShader.struct.MyParams( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i32 [[TMP0:%.*]], ptr [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_MYPARAMS:%.*]], ptr [[TMP1]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP4]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = call ptr inttoptr (i64 2 to ptr)([[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]]), !continuation.registercount !16, !continuation.returnedRegistercount !16 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @await.struct.DispatchSystemData(ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_MYPARAMS]] poison, ptr [[TMP1]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_MYPARAMS]], ptr [[TMP1]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = load i32, ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP8]], ptr [[TMP7]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP6]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_CallShader( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i32 [[TMP0:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[NEWDATA:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @_AmdAwaitShader(i64 2, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; -; ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.DispatchSystemData @called( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !lgc.rt.shaderstage !17 !continuation.registercount !16 !continuation !18 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !lgc.rt.shaderstage [[META16:![0-9]+]] !continuation.registercount [[META17:![0-9]+]] !continuation [[META18:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_MYPARAMS:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP3]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_MYPARAMS]], ptr [[TMP2]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = load i32, ptr @PAYLOAD, align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP5]], ptr [[TMP4]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_CallShader.struct.MyParams(ptr [[SYSTEM_DATA_ALLOCA]], i32 2, ptr [[TMP2]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) +; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA_I:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_MYPARAMS]], ptr [[TMP2]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP7]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP8]], !continuation.registercount !16 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = call ptr inttoptr (i64 2 to ptr)([[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I]]), !continuation.registercount [[META17]], !continuation.returnedRegistercount !17 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @await.struct.DispatchSystemData(ptr [[TMP8]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_MYPARAMS]] poison, ptr [[TMP2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_MYPARAMS]], ptr [[TMP2]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = load i32, ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP11]], ptr [[TMP10]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP9]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; LOWERRAYTRACINGPIPELINE-NEXT: br label [[DOTSPLIT:%.*]] +; LOWERRAYTRACINGPIPELINE: .split: +; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_MYPARAMS]], ptr [[TMP2]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP13]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP14]], !continuation.registercount [[META17]] ; ; ; CLEANUP-LABEL: define i32 @_cont_GetLocalRootIndex( @@ -151,10 +133,9 @@ attributes #1 = { alwaysinline } ; ; ; CLEANUP-LABEL: define void @called( -; CLEANUP-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !lgc.rt.shaderstage !17 !continuation.registercount !18 !continuation !19 !continuation.state !20 !continuation.stacksize !20 { +; CLEANUP-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !lgc.rt.shaderstage [[META16:![0-9]+]] !continuation.registercount [[META17:![0-9]+]] !continuation [[META18:![0-9]+]] !continuation.state [[META19:![0-9]+]] !continuation.stacksize [[META19]] { ; CLEANUP-NEXT: AllocaSpillBB: ; CLEANUP-NEXT: [[CONT_STATE:%.*]] = alloca [2 x i32], align 4 -; CLEANUP-NEXT: call void @continuation.save.continuation_state() ; CLEANUP-NEXT: [[RETURNADDR_SPILL_ADDR:%.*]] = getelementptr inbounds [[CALLED_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; CLEANUP-NEXT: store i64 [[RETURNADDR]], ptr [[RETURNADDR_SPILL_ADDR]], align 4 ; CLEANUP-NEXT: [[TMP1:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() @@ -168,48 +149,52 @@ attributes #1 = { alwaysinline } ; CLEANUP-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4 ; CLEANUP-NEXT: [[TMP5:%.*]] = add i32 [[TMP4]], 8 ; CLEANUP-NEXT: store i32 [[TMP5]], ptr [[TMP3]], align 4 -; CLEANUP-NEXT: [[TMP6:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP6]]) +; CLEANUP-NEXT: [[TMP6:%.*]] = inttoptr i32 [[TMP4]] to ptr addrspace(21) ; CLEANUP-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CLEANUP-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4 -; CLEANUP-NEXT: store i32 [[TMP8]], ptr @CONTINUATION_STATE, align 4 -; CLEANUP-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CLEANUP-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4 -; CLEANUP-NEXT: store i32 [[TMP10]], ptr getelementptr inbounds ([2 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CLEANUP-NEXT: [[TMP11:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4 -; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 2, i32 [[TMP12]], i64 ptrtoint (ptr @called.resume.0 to i64), [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I_FCA_0_INSERT]]), !continuation.registercount !18, !continuation.returnedRegistercount !18 +; CLEANUP-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP6]], i32 0, i32 0 +; CLEANUP-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4 +; CLEANUP-NEXT: store i32 [[TMP9]], ptr addrspace(21) [[TMP8]], align 4 +; CLEANUP-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CLEANUP-NEXT: [[TMP11:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP6]], i32 0, i32 1 +; CLEANUP-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP10]], align 4 +; CLEANUP-NEXT: store i32 [[TMP12]], ptr addrspace(21) [[TMP11]], align 4 +; CLEANUP-NEXT: [[TMP13:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP14:%.*]] = load i32, ptr [[TMP13]], align 4 +; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 2, i32 [[TMP14]], i64 ptrtoint (ptr @called.resume.0 to i64), [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I_FCA_0_INSERT]]), !continuation.registercount [[META17]], !continuation.returnedRegistercount !17 ; CLEANUP-NEXT: unreachable ; ; -; CLEANUP-LABEL: define void @called.resume.0( -; CLEANUP-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage !17 !continuation.registercount !18 !continuation !19 { +; CLEANUP-LABEL: define dso_local void @called.resume.0( +; CLEANUP-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage [[META16]] !continuation.registercount [[META17]] !continuation [[META18]] { ; CLEANUP-NEXT: entryresume.0: ; CLEANUP-NEXT: [[CONT_STATE:%.*]] = alloca [2 x i32], align 4 -; CLEANUP-NEXT: [[TMP2:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CLEANUP-NEXT: [[TMP3:%.*]] = load i32, ptr @CONTINUATION_STATE, align 4 -; CLEANUP-NEXT: store i32 [[TMP3]], ptr [[TMP2]], align 4 -; CLEANUP-NEXT: [[TMP4:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CLEANUP-NEXT: [[TMP5:%.*]] = load i32, ptr getelementptr inbounds ([2 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CLEANUP-NEXT: store i32 [[TMP5]], ptr [[TMP4]], align 4 -; CLEANUP-NEXT: [[TMP6:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP6]]) -; CLEANUP-NEXT: [[TMP7:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4 -; CLEANUP-NEXT: [[TMP9:%.*]] = add i32 [[TMP8]], -8 -; CLEANUP-NEXT: store i32 [[TMP9]], ptr [[TMP7]], align 4 -; CLEANUP-NEXT: [[TMP10:%.*]] = load i32, ptr @PAYLOAD, align 4 +; CLEANUP-NEXT: [[TMP2:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4 +; CLEANUP-NEXT: [[TMP4:%.*]] = inttoptr i32 [[TMP3]] to ptr addrspace(21) +; CLEANUP-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP4]], i64 -8 +; CLEANUP-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP5]], i32 0, i32 0 +; CLEANUP-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; CLEANUP-NEXT: [[TMP8:%.*]] = load i32, ptr addrspace(21) [[TMP6]], align 4 +; CLEANUP-NEXT: store i32 [[TMP8]], ptr [[TMP7]], align 4 +; CLEANUP-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP5]], i32 0, i32 1 +; CLEANUP-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CLEANUP-NEXT: [[TMP11:%.*]] = load i32, ptr addrspace(21) [[TMP9]], align 4 +; CLEANUP-NEXT: store i32 [[TMP11]], ptr [[TMP10]], align 4 +; CLEANUP-NEXT: [[TMP12:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4 +; CLEANUP-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], -8 +; CLEANUP-NEXT: store i32 [[TMP14]], ptr [[TMP12]], align 4 +; CLEANUP-NEXT: [[TMP15:%.*]] = load i32, ptr @PAYLOAD, align 4 ; CLEANUP-NEXT: [[DOTFCA_0_EXTRACT3:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], 0 ; CLEANUP-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; CLEANUP-NEXT: [[RETURNADDR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[CALLED_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; CLEANUP-NEXT: [[RETURNADDR_RELOAD:%.*]] = load i64, ptr [[RETURNADDR_RELOAD_ADDR]], align 4 ; CLEANUP-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; CLEANUP-NEXT: store i32 [[TMP10]], ptr @PAYLOAD, align 4 +; CLEANUP-NEXT: store i32 [[TMP15]], ptr @PAYLOAD, align 4 ; CLEANUP-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, i32 [[DOTFCA_0_EXTRACT3]], 0 -; CLEANUP-NEXT: call void @continuation.restore.continuation_state() -; CLEANUP-NEXT: [[TMP11:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4 -; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP12]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount !18 +; CLEANUP-NEXT: [[TMP16:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4 +; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP17]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount [[META17]] ; CLEANUP-NEXT: unreachable ; ; @@ -219,85 +204,67 @@ attributes #1 = { alwaysinline } ; ; ; SAVESTATE-LABEL: define void @called( -; SAVESTATE-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !lgc.rt.shaderstage !15 !continuation.registercount !16 !continuation !17 !continuation.state !18 !continuation.stacksize !18 { +; SAVESTATE-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !lgc.rt.shaderstage [[META15:![0-9]+]] !continuation.registercount [[META16:![0-9]+]] !continuation [[META17:![0-9]+]] !continuation.state [[META18:![0-9]+]] !continuation.stacksize [[META18]] { ; SAVESTATE-NEXT: AllocaSpillBB: ; SAVESTATE-NEXT: [[CONT_STATE:%.*]] = alloca [2 x i32], align 4 ; SAVESTATE-NEXT: [[CSP:%.*]] = alloca i32, align 4 ; SAVESTATE-NEXT: store i32 [[CSPINIT]], ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP1:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 0 -; SAVESTATE-NEXT: store i32 [[TMP2]], ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP1]] to ptr addrspace(21) ; SAVESTATE-NEXT: [[RETURNADDR_SPILL_ADDR:%.*]] = getelementptr inbounds [[CALLED_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; SAVESTATE-NEXT: store i64 [[RETURNADDR]], ptr [[RETURNADDR_SPILL_ADDR]], align 4 -; SAVESTATE-NEXT: [[TMP4:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() -; SAVESTATE-NEXT: [[DOTFCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP4]], 0 +; SAVESTATE-NEXT: [[TMP1:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() +; SAVESTATE-NEXT: [[DOTFCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], 0 ; SAVESTATE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) -; SAVESTATE-NEXT: [[TMP5:%.*]] = load i32, ptr addrspace(20) @PAYLOAD, align 4 +; SAVESTATE-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(20) @PAYLOAD, align 4 ; SAVESTATE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr addrspacecast (ptr addrspace(20) @PAYLOAD to ptr)) ; SAVESTATE-NEXT: [[DIS_DATA_I_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, i32 [[DOTFCA_0_EXTRACT]], 0 -; SAVESTATE-NEXT: store i32 [[TMP5]], ptr addrspace(20) @PAYLOAD, align 4 -; SAVESTATE-NEXT: [[TMP6:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP7:%.*]] = add i32 [[TMP6]], 8 -; SAVESTATE-NEXT: store i32 [[TMP7]], ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; SAVESTATE-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4 -; SAVESTATE-NEXT: [[TMP10:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP11:%.*]] = inttoptr i32 [[TMP10]] to ptr addrspace(21) -; SAVESTATE-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP11]], i64 -8 -; SAVESTATE-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP12]], i32 0 -; SAVESTATE-NEXT: store i32 [[TMP9]], ptr addrspace(21) [[TMP13]], align 4 -; SAVESTATE-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; SAVESTATE-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4 -; SAVESTATE-NEXT: [[TMP16:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP17:%.*]] = inttoptr i32 [[TMP16]] to ptr addrspace(21) -; SAVESTATE-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP17]], i64 -8 -; SAVESTATE-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP18]], i32 0 -; SAVESTATE-NEXT: [[TMP20:%.*]] = getelementptr [2 x i32], ptr addrspace(21) [[TMP19]], i32 0, i32 1 -; SAVESTATE-NEXT: store i32 [[TMP15]], ptr addrspace(21) [[TMP20]], align 4 -; SAVESTATE-NEXT: [[TMP21:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: call void (i64, ...) @continuation.continue(i64 2, i32 [[TMP21]], i64 ptrtoint (ptr @called.resume.0 to i64), [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I_FCA_0_INSERT]]), !continuation.registercount !16, !continuation.returnedRegistercount !16 +; SAVESTATE-NEXT: store i32 [[TMP2]], ptr addrspace(20) @PAYLOAD, align 4 +; SAVESTATE-NEXT: [[TMP3:%.*]] = load i32, ptr [[CSP]], align 4 +; SAVESTATE-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 8 +; SAVESTATE-NEXT: store i32 [[TMP4]], ptr [[CSP]], align 4 +; SAVESTATE-NEXT: [[TMP5:%.*]] = inttoptr i32 [[TMP3]] to ptr addrspace(21) +; SAVESTATE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; SAVESTATE-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP5]], i32 0, i32 0 +; SAVESTATE-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP6]], align 4 +; SAVESTATE-NEXT: store i32 [[TMP8]], ptr addrspace(21) [[TMP7]], align 4 +; SAVESTATE-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; SAVESTATE-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP5]], i32 0, i32 1 +; SAVESTATE-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP9]], align 4 +; SAVESTATE-NEXT: store i32 [[TMP11]], ptr addrspace(21) [[TMP10]], align 4 +; SAVESTATE-NEXT: [[TMP12:%.*]] = load i32, ptr [[CSP]], align 4 +; SAVESTATE-NEXT: call void (i64, ...) @continuation.continue(i64 2, i32 [[TMP12]], i64 ptrtoint (ptr @called.resume.0 to i64), [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I_FCA_0_INSERT]]), !continuation.registercount [[META16]], !continuation.returnedRegistercount !16 ; SAVESTATE-NEXT: unreachable ; ; -; SAVESTATE-LABEL: define void @called.resume.0( -; SAVESTATE-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage !15 !continuation.registercount !16 !continuation !17 { +; SAVESTATE-LABEL: define dso_local void @called.resume.0( +; SAVESTATE-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage [[META15]] !continuation.registercount [[META16]] !continuation [[META17]] { ; SAVESTATE-NEXT: entryresume.0: ; SAVESTATE-NEXT: [[CONT_STATE:%.*]] = alloca [2 x i32], align 4 ; SAVESTATE-NEXT: [[CSP:%.*]] = alloca i32, align 4 ; SAVESTATE-NEXT: store i32 [[TMP0]], ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP2:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; SAVESTATE-NEXT: [[TMP3:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP4:%.*]] = inttoptr i32 [[TMP3]] to ptr addrspace(21) -; SAVESTATE-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP4]], i64 -8 -; SAVESTATE-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP5]], i32 0 -; SAVESTATE-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(21) [[TMP6]], align 4 -; SAVESTATE-NEXT: store i32 [[TMP7]], ptr [[TMP2]], align 4 -; SAVESTATE-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; SAVESTATE-NEXT: [[TMP9:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP10:%.*]] = inttoptr i32 [[TMP9]] to ptr addrspace(21) -; SAVESTATE-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP10]], i64 -8 -; SAVESTATE-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP11]], i32 0 -; SAVESTATE-NEXT: [[TMP13:%.*]] = getelementptr [2 x i32], ptr addrspace(21) [[TMP12]], i32 0, i32 1 -; SAVESTATE-NEXT: [[TMP14:%.*]] = load i32, ptr addrspace(21) [[TMP13]], align 4 -; SAVESTATE-NEXT: store i32 [[TMP14]], ptr [[TMP8]], align 4 -; SAVESTATE-NEXT: [[TMP15:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], -8 -; SAVESTATE-NEXT: store i32 [[TMP16]], ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP17:%.*]] = load i32, ptr addrspace(20) @PAYLOAD, align 4 +; SAVESTATE-NEXT: [[TMP2:%.*]] = load i32, ptr [[CSP]], align 4 +; SAVESTATE-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(21) +; SAVESTATE-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP3]], i64 -8 +; SAVESTATE-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 0 +; SAVESTATE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; SAVESTATE-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(21) [[TMP5]], align 4 +; SAVESTATE-NEXT: store i32 [[TMP7]], ptr [[TMP6]], align 4 +; SAVESTATE-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 1 +; SAVESTATE-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; SAVESTATE-NEXT: [[TMP10:%.*]] = load i32, ptr addrspace(21) [[TMP8]], align 4 +; SAVESTATE-NEXT: store i32 [[TMP10]], ptr [[TMP9]], align 4 +; SAVESTATE-NEXT: [[TMP11:%.*]] = load i32, ptr [[CSP]], align 4 +; SAVESTATE-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], -8 +; SAVESTATE-NEXT: store i32 [[TMP12]], ptr [[CSP]], align 4 +; SAVESTATE-NEXT: [[TMP13:%.*]] = load i32, ptr addrspace(20) @PAYLOAD, align 4 ; SAVESTATE-NEXT: [[DOTFCA_0_EXTRACT3:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], 0 ; SAVESTATE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; SAVESTATE-NEXT: [[RETURNADDR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[CALLED_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; SAVESTATE-NEXT: [[RETURNADDR_RELOAD:%.*]] = load i64, ptr [[RETURNADDR_RELOAD_ADDR]], align 4 ; SAVESTATE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr addrspacecast (ptr addrspace(20) @PAYLOAD to ptr)) -; SAVESTATE-NEXT: store i32 [[TMP17]], ptr addrspace(20) @PAYLOAD, align 4 +; SAVESTATE-NEXT: store i32 [[TMP13]], ptr addrspace(20) @PAYLOAD, align 4 ; SAVESTATE-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, i32 [[DOTFCA_0_EXTRACT3]], 0 -; SAVESTATE-NEXT: [[TMP18:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP19:%.*]] = add i32 [[TMP18]], 0 -; SAVESTATE-NEXT: store i32 [[TMP19]], ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP20:%.*]] = inttoptr i32 [[TMP19]] to ptr addrspace(21) -; SAVESTATE-NEXT: [[TMP21:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP21]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount !16 +; SAVESTATE-NEXT: [[TMP14:%.*]] = load i32, ptr [[CSP]], align 4 +; SAVESTATE-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP14]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount [[META16]] ; SAVESTATE-NEXT: unreachable ; ; @@ -306,50 +273,32 @@ attributes #1 = { alwaysinline } ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: ret i32 5 ; ; -; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define void @_cont_CallShader.struct.MyParams( -; LOWERRAYTRACINGPIPELINE-CPS-SAME: ptr [[DATA:%.*]], i32 [[TMP0:%.*]], ptr [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] { -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_MYPARAMS:%.*]], ptr [[TMP1]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP4]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP5:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] (...) @lgc.cps.await.s_struct.DispatchSystemDatas(i32 2, i32 2, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_MYPARAMS]] poison, ptr [[TMP1]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_MYPARAMS]], ptr [[TMP1]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP7:%.*]] = load i32, ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP7]], ptr [[TMP6]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP5]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define void @_cont_CallShader( -; LOWERRAYTRACINGPIPELINE-CPS-SAME: ptr [[DATA:%.*]], i32 [[TMP0:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[NEWDATA:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @_AmdAwaitShader(i64 2, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: ret void -; -; ; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define void @called( -; LOWERRAYTRACINGPIPELINE-CPS-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !lgc.rt.shaderstage !16 !lgc.cps !17 !continuation !18 { +; LOWERRAYTRACINGPIPELINE-CPS-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !lgc.rt.shaderstage [[META16:![0-9]+]] !lgc.cps [[META17:![0-9]+]] !continuation [[META18:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_MYPARAMS:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP3:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP3]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_CallShader.struct.MyParams(ptr [[SYSTEM_DATA_ALLOCA]], i32 2, ptr [[TMP2]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[DIS_DATA_I:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_MYPARAMS]], ptr [[TMP2]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP5]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP6:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 2, {} poison, [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP6]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP6:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] (...) @lgc.cps.await.s_struct.DispatchSystemDatas(i32 2, i32 2, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_MYPARAMS]] poison, ptr [[TMP2]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_MYPARAMS]], ptr [[TMP2]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP8:%.*]] = load i32, ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP8]], ptr [[TMP7]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP6]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br label [[DOTSPLIT:%.*]] +; LOWERRAYTRACINGPIPELINE-CPS: .split: +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_MYPARAMS]], ptr [[TMP2]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP10]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP11:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 2, {} poison, [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP11]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: unreachable ; ; @@ -359,7 +308,7 @@ attributes #1 = { alwaysinline } ; ; ; CLEANUP-CPS-LABEL: define void @called( -; CLEANUP-CPS-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !lgc.rt.shaderstage !16 !lgc.cps !17 !continuation !18 { +; CLEANUP-CPS-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) !lgc.rt.shaderstage [[META16:![0-9]+]] !lgc.cps [[META17:![0-9]+]] !continuation [[META18:![0-9]+]] { ; CLEANUP-CPS-NEXT: AllocaSpillBB: ; CLEANUP-CPS-NEXT: [[TMP1:%.*]] = call ptr addrspace(32) @lgc.cps.alloc(i32 8) ; CLEANUP-CPS-NEXT: [[RETURN_ADDR_SPILL_ADDR:%.*]] = getelementptr inbounds [[CALLED_FRAME:%.*]], ptr addrspace(32) [[TMP1]], i32 0, i32 0 @@ -374,8 +323,8 @@ attributes #1 = { alwaysinline } ; CLEANUP-CPS-NEXT: unreachable ; ; -; CLEANUP-CPS-LABEL: define void @called.resume.0( -; CLEANUP-CPS-SAME: {} [[TMP0:%.*]], i32 [[TMP1:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP2:%.*]]) !lgc.rt.shaderstage !16 !lgc.cps !17 !continuation !18 { +; CLEANUP-CPS-LABEL: define dso_local void @called.resume.0( +; CLEANUP-CPS-SAME: {} [[TMP0:%.*]], i32 [[TMP1:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP2:%.*]]) !lgc.rt.shaderstage [[META16]] !lgc.cps [[META17]] !continuation [[META18]] { ; CLEANUP-CPS-NEXT: entryresume.0: ; CLEANUP-CPS-NEXT: [[TMP3:%.*]] = call ptr addrspace(32) @lgc.cps.peek(i32 8) ; CLEANUP-CPS-NEXT: [[TMP4:%.*]] = load i32, ptr @PAYLOAD, align 4 diff --git a/shared/continuations/test/dx/lower-rt-pipeline-small-payload-field.ll b/shared/continuations/test/dx/lower-rt-pipeline-small-payload-field.ll index 11c3bc4c21..b8d80912d3 100644 --- a/shared/continuations/test/dx/lower-rt-pipeline-small-payload-field.ll +++ b/shared/continuations/test/dx/lower-rt-pipeline-small-payload-field.ll @@ -2,7 +2,7 @@ ; Test copying of fields between local and global payload whose size ; is not a multiple of i32s, requiring copies at a smaller granularity ; for at least a suffix of the fields. -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2>%t.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2> %t.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s ; RUN: count 0 < %t.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" @@ -110,8 +110,7 @@ attributes #3 = { nounwind memory(none) } ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = call [[STRUCT_SYSTEMDATA]] @continuations.getSystemData.s_struct.SystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_SYSTEMDATA]] [[TMP3]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP4]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_PAYLOAD]], ptr [[TMP2]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[TMP5]], i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i64 0 diff --git a/shared/continuations/test/dx/lower-rt-pipeline.ll b/shared/continuations/test/dx/lower-rt-pipeline.ll index 82b47e8afe..7645dd3100 100644 --- a/shared/continuations/test/dx/lower-rt-pipeline.ll +++ b/shared/continuations/test/dx/lower-rt-pipeline.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 3 -; RUN: grep -v lgc.cps.module %s | opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S 2>%t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s +; RUN: grep -v lgc.cps.module %s | opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S 2> %t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s ; RUN: count 0 < %t0.stderr -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2>%t1.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE-CPS %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2> %t1.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE-CPS %s ; RUN: count 0 < %t1.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" @@ -397,165 +397,6 @@ attributes #5 = { nocallback nofree nosync nounwind willreturn memory(argmem: re !62 = !{i32 0, %struct.BuiltInTriangleIntersectionAttributes2 poison} !63 = !{!"function", !"void", i64 poison, !64} !64 = !{i32 0, i8 poison} -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_TraceRay.struct.RayPayload.attr_max_32_bytes( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i64 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]], i32 [[TMP3:%.*]], i32 [[TMP4:%.*]], i32 [[TMP5:%.*]], float [[TMP6:%.*]], float [[TMP7:%.*]], float [[TMP8:%.*]], float [[TMP9:%.*]], float [[TMP10:%.*]], float [[TMP11:%.*]], float [[TMP12:%.*]], float [[TMP13:%.*]], ptr [[TMP14:%.*]]) #[[ATTR0:[0-9]+]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[ADDR:%.*]] = call i64 @_AmdGetResumePointAddr() #[[ATTR3:[0-9]+]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA2:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA]], i64 [[ADDR]], 5 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD:%.*]], ptr [[TMP14]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = getelementptr i32, ptr [[TMP16]], i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[TMP17]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP18]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP19]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[TMP16]], i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[TMP20]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP22]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP23:%.*]] = getelementptr i32, ptr [[TMP20]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP24]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr [[TMP20]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP25]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP26]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP27:%.*]] = call ptr inttoptr (i64 4 to ptr)([[STRUCT_TRAVERSALDATA]] [[TRAV_DATA2]]), !continuation.registercount !35, !continuation.returnedRegistercount !35 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @await.struct.DispatchSystemData(ptr [[TMP27]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_RAYPAYLOAD]] poison, ptr [[TMP14]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP14]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP30:%.*]] = getelementptr i32, ptr [[TMP29]], i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP31:%.*]] = getelementptr i32, ptr [[TMP30]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP32:%.*]] = load i32, ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP32]], ptr [[TMP31]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP33:%.*]] = getelementptr i32, ptr [[TMP29]], i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = getelementptr i32, ptr [[TMP33]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP35:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP35]], ptr [[TMP34]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP36:%.*]] = getelementptr i32, ptr [[TMP33]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP37:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP37]], ptr [[TMP36]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = getelementptr i32, ptr [[TMP33]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP39]], ptr [[TMP38]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP28]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_TraceRay( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i64 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]], i32 [[TMP3:%.*]], i32 [[TMP4:%.*]], i32 [[TMP5:%.*]], float [[TMP6:%.*]], float [[TMP7:%.*]], float [[TMP8:%.*]], float [[TMP9:%.*]], float [[TMP10:%.*]], float [[TMP11:%.*]], float [[TMP12:%.*]], float [[TMP13:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[ADDR:%.*]] = call i64 @_AmdGetResumePointAddr() #[[ATTR3]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA2:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA]], i64 [[ADDR]], 5 -; LOWERRAYTRACINGPIPELINE-NEXT: [[NEWDATA:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @_AmdAwaitTraversal(i64 4, [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA2]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_CallShader( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i32 [[TMP0:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[NEWDATA:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @_AmdAwaitShader(i64 2, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define i1 @_cont_ReportHit.struct.BuiltInTriangleIntersectionAttributes( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], float [[T:%.*]], i32 [[HITKIND:%.*]], ptr [[TMP0:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: [[ORIGTPTR:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA:%.*]], ptr [[DATA]], i32 0, i32 0, i32 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[ORIGT:%.*]] = load float, ptr [[ORIGTPTR]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[ISNOHIT:%.*]] = fcmp fast uge float [[T]], [[ORIGT]] -; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[ISNOHIT]], label [[ISEND:%.*]], label [[CALLAHIT:%.*]] -; LOWERRAYTRACINGPIPELINE: callAHit: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP0]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = call ptr inttoptr (i64 3 to ptr)([[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA]], float [[T]], i32 [[HITKIND]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP3]]), !continuation.registercount !33, !continuation.returnedRegistercount !33 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @await.struct.AnyHitTraversalData(ptr [[TMP4]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP5]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[DATA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP6]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemDataAnyHit(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret i1 true -; LOWERRAYTRACINGPIPELINE: isEnd: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP9]], ptr [[TMP8]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP10]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP12]], ptr [[TMP11]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP2]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[DATA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP14]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP13]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret i1 false -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define i1 @_cont_ReportHit.struct.BuiltInTriangleIntersectionAttributes2( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], float [[T:%.*]], i32 [[HITKIND:%.*]], ptr [[TMP0:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: [[ORIGTPTR:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA:%.*]], ptr [[DATA]], i32 0, i32 0, i32 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[ORIGT:%.*]] = load float, ptr [[ORIGTPTR]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[ISNOHIT:%.*]] = fcmp fast uge float [[T]], [[ORIGT]] -; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[ISNOHIT]], label [[ISEND:%.*]], label [[CALLAHIT:%.*]] -; LOWERRAYTRACINGPIPELINE: callAHit: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES2:%.*]], ptr [[TMP0]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = call ptr inttoptr (i64 3 to ptr)([[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA]], float [[T]], i32 [[HITKIND]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES2]] [[TMP3]]), !continuation.registercount !33, !continuation.returnedRegistercount !33 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @await.struct.AnyHitTraversalData(ptr [[TMP4]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP5]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[DATA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP6]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemDataAnyHit(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret i1 true -; LOWERRAYTRACINGPIPELINE: isEnd: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP9]], ptr [[TMP8]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP10]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP12]], ptr [[TMP11]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP2]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[DATA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP14]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP13]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret i1 false -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define i1 @_cont_ReportHit( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], float [[T:%.*]], i32 [[HITKIND:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[ORIGTPTR:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA:%.*]], ptr [[DATA]], i32 0, i32 0, i32 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[ORIGT:%.*]] = load float, ptr [[ORIGTPTR]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[ISNOHIT:%.*]] = fcmp fast uge float [[T]], [[ORIGT]] -; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[ISNOHIT]], label [[ISEND:%.*]], label [[CALLAHIT:%.*]] -; LOWERRAYTRACINGPIPELINE: callAHit: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[NEWDATA:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @_AmdAwaitAnyHit(i64 3, [[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA]], float [[T]], i32 [[HITKIND]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[DATA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP1]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemDataAnyHit(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret i1 true -; LOWERRAYTRACINGPIPELINE: isEnd: -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdAcceptHitAttributes(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret i1 false -; -; ; LOWERRAYTRACINGPIPELINE-LABEL: define <3 x i32> @_cont_DispatchRaysIndex3( ; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]]) { ; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_1:%.*]] = getelementptr [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], i32 0, i32 0, i32 0 @@ -606,7 +447,7 @@ attributes #5 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; ; ; LOWERRAYTRACINGPIPELINE-LABEL: define void @MyRayGen( -; LOWERRAYTRACINGPIPELINE-SAME: ) #[[ATTR2:[0-9]+]] !lgc.rt.shaderstage !23 !continuation.entry !14 !continuation.registercount !23 !continuation !36 { +; LOWERRAYTRACINGPIPELINE-SAME: ) #[[ATTR2:[0-9]+]] !lgc.rt.shaderstage [[META23:![0-9]+]] !continuation.entry [[META14:![0-9]+]] !continuation.registercount [[META23]] !continuation [[META35:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP1:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 @@ -618,29 +459,71 @@ attributes #5 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = bitcast ptr [[TMP4]] to ptr ; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr [[TMP5]]) #[[ATTR1:[0-9]+]] ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP4]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: store <4 x float> zeroinitializer, ptr [[TMP6]], align 4, !tbaa [[TBAA37:![0-9]+]] +; LOWERRAYTRACINGPIPELINE-NEXT: store <4 x float> zeroinitializer, ptr [[TMP6]], align 4, !tbaa [[TBAA36:![0-9]+]] ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP2]]) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP7]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[TMP8]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_TraceRay.struct.RayPayload.attr_max_32_bytes(ptr [[SYSTEM_DATA_ALLOCA]], i64 [[TMP9]], i32 16, i32 -1, i32 0, i32 1, i32 0, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0x3F50624DE0000000, float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 1.000000e+04, ptr [[TMP4]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = load <4 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA37]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() -; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT:%.*]] = extractelement <3 x i32> [[TMP11]], i8 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() -; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT1:%.*]] = extractelement <3 x i32> [[TMP12]], i8 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP3]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP13]], [[DX_TYPES_RESOURCEPROPERTIES]] { i32 4098, i32 1033 }) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = extractelement <4 x float> [[TMP10]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = extractelement <4 x float> [[TMP10]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = extractelement <4 x float> [[TMP10]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = extractelement <4 x float> [[TMP10]], i64 3 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @dx.op.textureStore.f32(i32 67, [[DX_TYPES_HANDLE]] [[TMP14]], i32 [[EXTRACT]], i32 [[EXTRACT1]], i32 undef, float [[TMP15]], float [[TMP16]], float [[TMP17]], float [[TMP18]], i8 15) +; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA_I:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA_I:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I]], 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA_I:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA_I]], 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[ADDR_I:%.*]] = call i64 @_AmdGetResumePointAddr() #[[ATTR3:[0-9]+]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA2_I:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]], i64 [[ADDR_I]], 5 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP4]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[TMP10]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[TMP11]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP13]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[TMP10]], i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[TMP14]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP16]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = getelementptr i32, ptr [[TMP14]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP18]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[TMP14]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP19]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP20]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP21:%.*]] = call ptr inttoptr (i64 4 to ptr)([[STRUCT_TRAVERSALDATA]] [[TRAV_DATA2_I]]), !continuation.registercount [[META39:![0-9]+]], !continuation.returnedRegistercount !39 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP22:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @await.struct.DispatchSystemData(ptr [[TMP21]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_RAYPAYLOAD]] poison, ptr [[TMP4]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP23:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP4]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP24:%.*]] = getelementptr i32, ptr [[TMP23]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr [[TMP24]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP26:%.*]] = load i32, ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP26]], ptr [[TMP25]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP27:%.*]] = getelementptr i32, ptr [[TMP23]], i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr [[TMP27]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP29]], ptr [[TMP28]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP30:%.*]] = getelementptr i32, ptr [[TMP27]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP31:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP31]], ptr [[TMP30]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr [[TMP27]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP33:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP33]], ptr [[TMP32]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP22]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX1:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[SYSTEM_DATA_ALLOCA]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX1]]) +; LOWERRAYTRACINGPIPELINE-NEXT: br label [[DOTSPLIT:%.*]] +; LOWERRAYTRACINGPIPELINE: .split: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = load <4 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA36]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP35:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() +; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT:%.*]] = extractelement <3 x i32> [[TMP35]], i8 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP36:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() +; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT1:%.*]] = extractelement <3 x i32> [[TMP36]], i8 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP37:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP3]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP37]], [[DX_TYPES_RESOURCEPROPERTIES]] { i32 4098, i32 1033 }) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = extractelement <4 x float> [[TMP34]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP40:%.*]] = extractelement <4 x float> [[TMP34]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP41:%.*]] = extractelement <4 x float> [[TMP34]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP42:%.*]] = extractelement <4 x float> [[TMP34]], i64 3 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @dx.op.textureStore.f32(i32 67, [[DX_TYPES_HANDLE]] [[TMP38]], i32 [[EXTRACT]], i32 [[EXTRACT1]], i32 undef, float [[TMP39]], float [[TMP40]], float [[TMP41]], float [[TMP42]], i8 15) ; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.end.p0(i64 16, ptr [[TMP5]]) #[[ATTR1]] -; LOWERRAYTRACINGPIPELINE-NEXT: ret void, !continuation.registercount !33 +; LOWERRAYTRACINGPIPELINE-NEXT: ret void, !continuation.registercount [[META33:![0-9]+]] ; ; ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.DispatchSystemData @MyClosestHitShader( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage !40 !continuation.registercount !35 !continuation !41 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage [[META40:![0-9]+]] !continuation.registercount [[META39]] !continuation [[META41:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_RAYPAYLOAD:%.*]], align 8 @@ -706,11 +589,11 @@ attributes #5 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP45]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP46:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP47:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP46]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP47]], !continuation.registercount !35 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP47]], !continuation.registercount [[META39]] ; ; ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.AnyHitTraversalData @MyAnyHitShader( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] [[TMP1:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage !42 !continuation.registercount !35 !continuation !43 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] [[TMP1:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage [[META42:![0-9]+]] !continuation.registercount [[META39]] !continuation [[META43:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_HITDATA:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = alloca [[STRUCT_HITDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = alloca [[STRUCT_HITDATA]], align 8 @@ -762,252 +645,326 @@ attributes #5 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP36:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP37:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP37]], ptr [[TMP4]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = call <3 x float> @_cont_ObjectRayOrigin3(ptr [[TMP36]], ptr [[TMP4]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT1:%.*]] = extractelement <3 x float> [[TMP38]], i8 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP40:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP40]], ptr [[TMP5]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP41:%.*]] = call <3 x float> @_cont_ObjectRayDirection3(ptr [[TMP39]], ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT:%.*]] = extractelement <3 x float> [[TMP41]], i8 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP42:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP43:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP43]], ptr [[TMP3]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP44:%.*]] = call float @_cont_RayTCurrent(ptr [[TMP42]], ptr [[TMP3]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP45:%.*]] = fmul fast float [[TMP44]], [[EXTRACT]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP46:%.*]] = fadd fast float [[TMP45]], [[EXTRACT1]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP47:%.*]] = fcmp fast ogt float [[TMP46]], 0.000000e+00 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP48:%.*]] = fcmp fast ogt float [[TMP46]], 1.000000e+00 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP49:%.*]] = fcmp fast ogt float [[TMP46]], -1.000000e+00 -; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[TMP47]], label [[TMP50:%.*]], label [[TMP95:%.*]] -; LOWERRAYTRACINGPIPELINE: 50: +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_1_I1:%.*]] = load float, ptr [[TMP4]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_2_I2:%.*]] = getelementptr [[STRUCT_HITDATA]], ptr [[TMP4]], i32 0, i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_2_I3:%.*]] = load float, ptr [[RESPTR_2_I2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_3_I4:%.*]] = getelementptr [[STRUCT_HITDATA]], ptr [[TMP4]], i32 0, i32 0, i32 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_3_I5:%.*]] = load float, ptr [[RESPTR_3_I4]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[VAL_0_I6:%.*]] = insertelement <3 x float> undef, float [[RES_1_I1]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[VAL_1_I7:%.*]] = insertelement <3 x float> [[VAL_0_I6]], float [[RES_2_I3]], i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[VAL_2_I8:%.*]] = insertelement <3 x float> [[VAL_1_I7]], float [[RES_3_I5]], i32 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT1:%.*]] = extractelement <3 x float> [[VAL_2_I8]], i8 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP39]], ptr [[TMP5]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_1_I:%.*]] = getelementptr [[STRUCT_HITDATA]], ptr [[TMP5]], i32 0, i32 1, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_1_I:%.*]] = load float, ptr [[RESPTR_1_I]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_2_I:%.*]] = getelementptr [[STRUCT_HITDATA]], ptr [[TMP5]], i32 0, i32 1, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_2_I:%.*]] = load float, ptr [[RESPTR_2_I]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_3_I:%.*]] = getelementptr [[STRUCT_HITDATA]], ptr [[TMP5]], i32 0, i32 1, i32 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_3_I:%.*]] = load float, ptr [[RESPTR_3_I]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[VAL_0_I:%.*]] = insertelement <3 x float> undef, float [[RES_1_I]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[VAL_1_I:%.*]] = insertelement <3 x float> [[VAL_0_I]], float [[RES_2_I]], i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[VAL_2_I:%.*]] = insertelement <3 x float> [[VAL_1_I]], float [[RES_3_I]], i32 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT:%.*]] = extractelement <3 x float> [[VAL_2_I]], i8 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP40:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP41:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP41]], ptr [[TMP3]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_I:%.*]] = getelementptr [[STRUCT_HITDATA]], ptr [[TMP3]], i32 0, i32 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I:%.*]] = load float, ptr [[RESPTR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP42:%.*]] = fmul fast float [[RES_I]], [[EXTRACT]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP43:%.*]] = fadd fast float [[TMP42]], [[EXTRACT1]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP44:%.*]] = fcmp fast ogt float [[TMP43]], 0.000000e+00 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP45:%.*]] = fcmp fast ogt float [[TMP43]], 1.000000e+00 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP46:%.*]] = fcmp fast ogt float [[TMP43]], -1.000000e+00 +; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[TMP44]], label [[TMP47:%.*]], label [[TMP92:%.*]] +; LOWERRAYTRACINGPIPELINE: 47: ; LOWERRAYTRACINGPIPELINE-NEXT: store <4 x float> [[TMP35]], ptr [[TMP34]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[TMP48]], label [[TMP51:%.*]], label [[TMP73:%.*]] -; LOWERRAYTRACINGPIPELINE: 51: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP52:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_AcceptHitAndEndSearch(ptr [[TMP52]]) +; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[TMP45]], label [[TMP48:%.*]], label [[TMP70:%.*]] +; LOWERRAYTRACINGPIPELINE: 48: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP49:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_AcceptHitAndEndSearch(ptr [[TMP49]]) ; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP53:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP54:%.*]] = getelementptr i32, ptr [[TMP53]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP50:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP51:%.*]] = getelementptr i32, ptr [[TMP50]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP52:%.*]] = getelementptr i32, ptr [[TMP51]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP53:%.*]] = load i32, ptr [[TMP52]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP53]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP54:%.*]] = getelementptr i32, ptr [[TMP50]], i32 1 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP55:%.*]] = getelementptr i32, ptr [[TMP54]], i64 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP56:%.*]] = load i32, ptr [[TMP55]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP56]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP57:%.*]] = getelementptr i32, ptr [[TMP53]], i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP58:%.*]] = getelementptr i32, ptr [[TMP57]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP59:%.*]] = load i32, ptr [[TMP58]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP59]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP60:%.*]] = getelementptr i32, ptr [[TMP57]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP61:%.*]] = load i32, ptr [[TMP60]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP61]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP62:%.*]] = getelementptr i32, ptr [[TMP57]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP63:%.*]] = load i32, ptr [[TMP62]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP63]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP64:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP65:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP56]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP57:%.*]] = getelementptr i32, ptr [[TMP54]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP58:%.*]] = load i32, ptr [[TMP57]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP58]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP59:%.*]] = getelementptr i32, ptr [[TMP54]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP60:%.*]] = load i32, ptr [[TMP59]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP60]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP61:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP62:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP63:%.*]] = load i32, ptr [[TMP61]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP63]], ptr [[TMP62]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP64:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP65:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 1 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP66:%.*]] = load i32, ptr [[TMP64]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP66]], ptr [[TMP65]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP67:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP68:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP69:%.*]] = load i32, ptr [[TMP67]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP69]], ptr [[TMP68]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP70:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP10]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP71:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP71]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP70]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP72:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP72]], !continuation.registercount !35 -; LOWERRAYTRACINGPIPELINE: 73: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP74:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_AcceptHitAndEndSearch(ptr [[TMP74]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP67:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP10]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP68:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP68]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP67]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP69:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP69]], !continuation.registercount [[META39]] +; LOWERRAYTRACINGPIPELINE: 70: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP71:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_AcceptHitAndEndSearch(ptr [[TMP71]]) ; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP75:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP76:%.*]] = getelementptr i32, ptr [[TMP75]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP72:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP73:%.*]] = getelementptr i32, ptr [[TMP72]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP74:%.*]] = getelementptr i32, ptr [[TMP73]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP75:%.*]] = load i32, ptr [[TMP74]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP75]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP76:%.*]] = getelementptr i32, ptr [[TMP72]], i32 1 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP77:%.*]] = getelementptr i32, ptr [[TMP76]], i64 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP78:%.*]] = load i32, ptr [[TMP77]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP78]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP79:%.*]] = getelementptr i32, ptr [[TMP75]], i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP80:%.*]] = getelementptr i32, ptr [[TMP79]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP81:%.*]] = load i32, ptr [[TMP80]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP81]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP82:%.*]] = getelementptr i32, ptr [[TMP79]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP83:%.*]] = load i32, ptr [[TMP82]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP83]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP84:%.*]] = getelementptr i32, ptr [[TMP79]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP85:%.*]] = load i32, ptr [[TMP84]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP85]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP86:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP87:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP78]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP79:%.*]] = getelementptr i32, ptr [[TMP76]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP80:%.*]] = load i32, ptr [[TMP79]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP80]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP81:%.*]] = getelementptr i32, ptr [[TMP76]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP82:%.*]] = load i32, ptr [[TMP81]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP82]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP83:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP84:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP85:%.*]] = load i32, ptr [[TMP83]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP85]], ptr [[TMP84]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP86:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP87:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i64 1 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP88:%.*]] = load i32, ptr [[TMP86]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP88]], ptr [[TMP87]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP89:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP90:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP91:%.*]] = load i32, ptr [[TMP89]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP91]], ptr [[TMP90]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP92:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP9]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP93:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP93]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP92]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP94:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP94]], !continuation.registercount !35 -; LOWERRAYTRACINGPIPELINE: 95: -; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[TMP49]], label [[TMP96:%.*]], label [[TMP141:%.*]] -; LOWERRAYTRACINGPIPELINE: 96: -; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[TMP48]], label [[TMP97:%.*]], label [[TMP119:%.*]] -; LOWERRAYTRACINGPIPELINE: 97: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP98:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_IgnoreHit(ptr [[TMP98]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP89:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP9]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP90:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP90]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP89]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP91:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP91]], !continuation.registercount [[META39]] +; LOWERRAYTRACINGPIPELINE: 92: +; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[TMP46]], label [[TMP93:%.*]], label [[TMP138:%.*]] +; LOWERRAYTRACINGPIPELINE: 93: +; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[TMP45]], label [[TMP94:%.*]], label [[TMP116:%.*]] +; LOWERRAYTRACINGPIPELINE: 94: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP95:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_IgnoreHit(ptr [[TMP95]]) ; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP99:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP100:%.*]] = getelementptr i32, ptr [[TMP99]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP96:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP97:%.*]] = getelementptr i32, ptr [[TMP96]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP98:%.*]] = getelementptr i32, ptr [[TMP97]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP99:%.*]] = load i32, ptr [[TMP98]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP99]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP100:%.*]] = getelementptr i32, ptr [[TMP96]], i32 1 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP101:%.*]] = getelementptr i32, ptr [[TMP100]], i64 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP102:%.*]] = load i32, ptr [[TMP101]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP102]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP103:%.*]] = getelementptr i32, ptr [[TMP99]], i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP104:%.*]] = getelementptr i32, ptr [[TMP103]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP105:%.*]] = load i32, ptr [[TMP104]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP105]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP106:%.*]] = getelementptr i32, ptr [[TMP103]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP107:%.*]] = load i32, ptr [[TMP106]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP107]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP108:%.*]] = getelementptr i32, ptr [[TMP103]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP109:%.*]] = load i32, ptr [[TMP108]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP109]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP110:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP111:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP102]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP103:%.*]] = getelementptr i32, ptr [[TMP100]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP104:%.*]] = load i32, ptr [[TMP103]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP104]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP105:%.*]] = getelementptr i32, ptr [[TMP100]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP106:%.*]] = load i32, ptr [[TMP105]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP106]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP107:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP108:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP109:%.*]] = load i32, ptr [[TMP107]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP109]], ptr [[TMP108]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP110:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP111:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 1 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP112:%.*]] = load i32, ptr [[TMP110]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP112]], ptr [[TMP111]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP113:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP114:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP115:%.*]] = load i32, ptr [[TMP113]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP115]], ptr [[TMP114]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP116:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP8]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP117:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP117]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP116]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP118:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP118]], !continuation.registercount !35 -; LOWERRAYTRACINGPIPELINE: 119: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP120:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_IgnoreHit(ptr [[TMP120]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP113:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP8]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP114:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP114]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP113]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP115:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP115]], !continuation.registercount [[META39]] +; LOWERRAYTRACINGPIPELINE: 116: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP117:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_IgnoreHit(ptr [[TMP117]]) ; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP121:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP122:%.*]] = getelementptr i32, ptr [[TMP121]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP118:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP119:%.*]] = getelementptr i32, ptr [[TMP118]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP120:%.*]] = getelementptr i32, ptr [[TMP119]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP121:%.*]] = load i32, ptr [[TMP120]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP121]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP122:%.*]] = getelementptr i32, ptr [[TMP118]], i32 1 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP123:%.*]] = getelementptr i32, ptr [[TMP122]], i64 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP124:%.*]] = load i32, ptr [[TMP123]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP124]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP125:%.*]] = getelementptr i32, ptr [[TMP121]], i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP126:%.*]] = getelementptr i32, ptr [[TMP125]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP127:%.*]] = load i32, ptr [[TMP126]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP127]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP128:%.*]] = getelementptr i32, ptr [[TMP125]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP129:%.*]] = load i32, ptr [[TMP128]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP129]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP130:%.*]] = getelementptr i32, ptr [[TMP125]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP131:%.*]] = load i32, ptr [[TMP130]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP131]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP132:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP133:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP124]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP125:%.*]] = getelementptr i32, ptr [[TMP122]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP126:%.*]] = load i32, ptr [[TMP125]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP126]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP127:%.*]] = getelementptr i32, ptr [[TMP122]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP128:%.*]] = load i32, ptr [[TMP127]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP128]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP129:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP130:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP131:%.*]] = load i32, ptr [[TMP129]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP131]], ptr [[TMP130]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP132:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP133:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 1 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP134:%.*]] = load i32, ptr [[TMP132]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP134]], ptr [[TMP133]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP135:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP136:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP137:%.*]] = load i32, ptr [[TMP135]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP137]], ptr [[TMP136]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP138:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP7]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP139:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP139]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP138]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP140:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP140]], !continuation.registercount !35 -; LOWERRAYTRACINGPIPELINE: 141: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP135:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP7]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP136:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP136]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP135]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP137:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP137]], !continuation.registercount [[META39]] +; LOWERRAYTRACINGPIPELINE: 138: ; LOWERRAYTRACINGPIPELINE-NEXT: store <4 x float> [[TMP35]], ptr [[TMP34]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_AcceptHit(ptr [[SYSTEM_DATA_ALLOCA]]) ; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP142:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP143:%.*]] = getelementptr i32, ptr [[TMP142]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP139:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP140:%.*]] = getelementptr i32, ptr [[TMP139]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP141:%.*]] = getelementptr i32, ptr [[TMP140]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP142:%.*]] = load i32, ptr [[TMP141]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP142]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP143:%.*]] = getelementptr i32, ptr [[TMP139]], i32 1 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP144:%.*]] = getelementptr i32, ptr [[TMP143]], i64 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP145:%.*]] = load i32, ptr [[TMP144]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP145]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP146:%.*]] = getelementptr i32, ptr [[TMP142]], i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP147:%.*]] = getelementptr i32, ptr [[TMP146]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP148:%.*]] = load i32, ptr [[TMP147]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP148]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP149:%.*]] = getelementptr i32, ptr [[TMP146]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP150:%.*]] = load i32, ptr [[TMP149]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP150]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP151:%.*]] = getelementptr i32, ptr [[TMP146]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP152:%.*]] = load i32, ptr [[TMP151]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP152]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP153:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP154:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP145]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP146:%.*]] = getelementptr i32, ptr [[TMP143]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP147:%.*]] = load i32, ptr [[TMP146]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP147]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP148:%.*]] = getelementptr i32, ptr [[TMP143]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP149:%.*]] = load i32, ptr [[TMP148]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP149]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP150:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP151:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP152:%.*]] = load i32, ptr [[TMP150]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP152]], ptr [[TMP151]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP153:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP154:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 1 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP155:%.*]] = load i32, ptr [[TMP153]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP155]], ptr [[TMP154]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP156:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP157:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP158:%.*]] = load i32, ptr [[TMP156]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP158]], ptr [[TMP157]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP159:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP6]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP160:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP160]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP159]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP161:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP161]], !continuation.registercount !35 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP156:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP6]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP157:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP157]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP156]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP158:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP158]], !continuation.registercount [[META39]] ; ; ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.AnyHitTraversalData @MyIntersectionShader( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage !44 !continuation.registercount !33 !continuation !45 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage [[META44:![0-9]+]] !continuation.registercount [[META33]] !continuation [[META45:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_HITDATA:%.*]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_ANYHITTRAVERSALDATA]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @continuations.getSystemData.s_struct.AnyHitTraversalDatas() -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP4]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @continuations.getSystemData.s_struct.AnyHitTraversalDatas() +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP5]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP7]], ptr [[TMP2]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = call float @_cont_RayTCurrent(ptr [[TMP6]], ptr [[TMP2]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = bitcast ptr [[TMP3]] to ptr +; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP6]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP8]], ptr [[TMP2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_I:%.*]] = getelementptr [[STRUCT_HITDATA]], ptr [[TMP2]], i32 0, i32 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I:%.*]] = load float, ptr [[RESPTR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = bitcast ptr [[TMP4]] to ptr ; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[TMP9]]) #[[ATTR1]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = call i1 @_cont_ReportHit.struct.BuiltInTriangleIntersectionAttributes(ptr [[SYSTEM_DATA_ALLOCA]], float [[TMP8]], i32 0, ptr [[TMP3]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = call i1 @_cont_IsEndSearch(ptr [[TMP11]]) -; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[TMP12]], label [[TMP13:%.*]], label [[TMP15:%.*]] -; LOWERRAYTRACINGPIPELINE: 13: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP14]], !continuation.registercount !33 -; LOWERRAYTRACINGPIPELINE: 15: +; LOWERRAYTRACINGPIPELINE-NEXT: [[ORIGTPTR_I:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[ORIGT_I:%.*]] = load float, ptr [[ORIGTPTR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[ISNOHIT_I:%.*]] = fcmp fast uge float [[RES_I]], [[ORIGT_I]] +; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[ISNOHIT_I]], label [[ISEND_I:%.*]], label [[CALLAHIT_I:%.*]] +; LOWERRAYTRACINGPIPELINE: callAHit.i: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA_I:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP4]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = call ptr inttoptr (i64 3 to ptr)([[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA_I]], float [[RES_I]], i32 0, [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP10]]), !continuation.registercount [[META33]], !continuation.returnedRegistercount !33 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @await.struct.AnyHitTraversalData(ptr [[TMP11]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP12]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX1:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP13]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX1]]) +; LOWERRAYTRACINGPIPELINE-NEXT: br label [[_CONT_REPORTHIT_EXIT:%.*]] +; LOWERRAYTRACINGPIPELINE: isEnd.i: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP14]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP16]], ptr [[TMP15]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP17]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP19]], ptr [[TMP18]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP20:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP3]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP21]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP20]]) +; LOWERRAYTRACINGPIPELINE-NEXT: br label [[_CONT_REPORTHIT_EXIT]] +; LOWERRAYTRACINGPIPELINE: _cont_ReportHit.exit: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP22:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP23:%.*]] = call i1 @_cont_IsEndSearch(ptr [[TMP22]]) +; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[TMP23]], label [[TMP24:%.*]], label [[TMP26:%.*]] +; LOWERRAYTRACINGPIPELINE: 24: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP25:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP25]], !continuation.registercount [[META33]] +; LOWERRAYTRACINGPIPELINE: 26: ; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[TMP9]]) #[[ATTR1]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP16]], !continuation.registercount !33 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP27:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP27]], !continuation.registercount [[META33]] ; ; ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.AnyHitTraversalData @MyIntersectionShader2( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage !44 !continuation.registercount !33 !continuation !46 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage [[META44]] !continuation.registercount [[META33]] !continuation [[META46:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_HITDATA:%.*]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES2:%.*]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES2:%.*]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_ANYHITTRAVERSALDATA]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @continuations.getSystemData.s_struct.AnyHitTraversalDatas() -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP4]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @continuations.getSystemData.s_struct.AnyHitTraversalDatas() +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP5]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP7]], ptr [[TMP2]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = call float @_cont_RayTCurrent(ptr [[TMP6]], ptr [[TMP2]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = bitcast ptr [[TMP3]] to ptr +; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP6]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP8]], ptr [[TMP2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_I:%.*]] = getelementptr [[STRUCT_HITDATA]], ptr [[TMP2]], i32 0, i32 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I:%.*]] = load float, ptr [[RESPTR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = bitcast ptr [[TMP4]] to ptr ; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[TMP9]]) #[[ATTR1]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = call i1 @_cont_ReportHit.struct.BuiltInTriangleIntersectionAttributes2(ptr [[SYSTEM_DATA_ALLOCA]], float [[TMP8]], i32 0, ptr [[TMP3]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = call i1 @_cont_IsEndSearch(ptr [[TMP11]]) -; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[TMP12]], label [[TMP13:%.*]], label [[TMP15:%.*]] -; LOWERRAYTRACINGPIPELINE: 13: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP14]], !continuation.registercount !33 -; LOWERRAYTRACINGPIPELINE: 15: +; LOWERRAYTRACINGPIPELINE-NEXT: [[ORIGTPTR_I:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[ORIGT_I:%.*]] = load float, ptr [[ORIGTPTR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[ISNOHIT_I:%.*]] = fcmp fast uge float [[RES_I]], [[ORIGT_I]] +; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[ISNOHIT_I]], label [[ISEND_I:%.*]], label [[CALLAHIT_I:%.*]] +; LOWERRAYTRACINGPIPELINE: callAHit.i: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA_I:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES2]], ptr [[TMP4]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = call ptr inttoptr (i64 3 to ptr)([[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA_I]], float [[RES_I]], i32 0, [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES2]] [[TMP10]]), !continuation.registercount [[META33]], !continuation.returnedRegistercount !33 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @await.struct.AnyHitTraversalData(ptr [[TMP11]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP12]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX1:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP13]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX1]]) +; LOWERRAYTRACINGPIPELINE-NEXT: br label [[_CONT_REPORTHIT_EXIT:%.*]] +; LOWERRAYTRACINGPIPELINE: isEnd.i: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP14]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP16]], ptr [[TMP15]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP17]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP19]], ptr [[TMP18]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP20:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP3]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP21]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP20]]) +; LOWERRAYTRACINGPIPELINE-NEXT: br label [[_CONT_REPORTHIT_EXIT]] +; LOWERRAYTRACINGPIPELINE: _cont_ReportHit.exit: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP22:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP23:%.*]] = call i1 @_cont_IsEndSearch(ptr [[TMP22]]) +; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[TMP23]], label [[TMP24:%.*]], label [[TMP26:%.*]] +; LOWERRAYTRACINGPIPELINE: 24: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP25:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP25]], !continuation.registercount [[META33]] +; LOWERRAYTRACINGPIPELINE: 26: ; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[TMP9]]) #[[ATTR1]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP16]], !continuation.registercount !33 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP27:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP27]], !continuation.registercount [[META33]] ; ; ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.DispatchSystemData @MyMissShader( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage !47 !continuation.registercount !35 !continuation !48 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage [[META47:![0-9]+]] !continuation.registercount [[META39]] !continuation [[META48:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_RAYPAYLOAD:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = call [[STRUCT_SYSTEMDATA]] @continuations.getSystemData.s_struct.SystemDatas() @@ -1051,163 +1008,7 @@ attributes #5 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP27]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP28]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP29]], !continuation.registercount !35 -; -; -; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define void @_cont_TraceRay.struct.RayPayload.attr_max_32_bytes( -; LOWERRAYTRACINGPIPELINE-CPS-SAME: ptr [[DATA:%.*]], i64 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]], i32 [[TMP3:%.*]], i32 [[TMP4:%.*]], i32 [[TMP5:%.*]], float [[TMP6:%.*]], float [[TMP7:%.*]], float [[TMP8:%.*]], float [[TMP9:%.*]], float [[TMP10:%.*]], float [[TMP11:%.*]], float [[TMP12:%.*]], float [[TMP13:%.*]], ptr [[TMP14:%.*]]) #[[ATTR0:[0-9]+]] { -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[SYS_DATA:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TRAV_DATA:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[ADDR:%.*]] = call i64 @_AmdGetResumePointAddr() #[[ATTR3:[0-9]+]] -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TRAV_DATA2:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA]], i64 [[ADDR]], 5 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP16:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD:%.*]], ptr [[TMP14]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP17:%.*]] = getelementptr i32, ptr [[TMP16]], i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[TMP17]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP18]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP19]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[TMP16]], i32 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[TMP20]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP22]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP23:%.*]] = getelementptr i32, ptr [[TMP20]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP24]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr [[TMP20]], i64 2 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP25]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP26]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP27:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] (...) @lgc.cps.await.s_struct.DispatchSystemDatas(i32 4, i32 4, [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA2]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_RAYPAYLOAD]] poison, ptr [[TMP14]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP28:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP14]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP29:%.*]] = getelementptr i32, ptr [[TMP28]], i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP30:%.*]] = getelementptr i32, ptr [[TMP29]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP31:%.*]] = load i32, ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP31]], ptr [[TMP30]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr [[TMP28]], i32 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP33:%.*]] = getelementptr i32, ptr [[TMP32]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP34:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP34]], ptr [[TMP33]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP35:%.*]] = getelementptr i32, ptr [[TMP32]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP36:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP36]], ptr [[TMP35]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP37:%.*]] = getelementptr i32, ptr [[TMP32]], i64 2 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP38:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP38]], ptr [[TMP37]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP27]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define void @_cont_TraceRay( -; LOWERRAYTRACINGPIPELINE-CPS-SAME: ptr [[DATA:%.*]], i64 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]], i32 [[TMP3:%.*]], i32 [[TMP4:%.*]], i32 [[TMP5:%.*]], float [[TMP6:%.*]], float [[TMP7:%.*]], float [[TMP8:%.*]], float [[TMP9:%.*]], float [[TMP10:%.*]], float [[TMP11:%.*]], float [[TMP12:%.*]], float [[TMP13:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[SYS_DATA:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TRAV_DATA:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[ADDR:%.*]] = call i64 @_AmdGetResumePointAddr() #[[ATTR3]] -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TRAV_DATA2:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA]], i64 [[ADDR]], 5 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[NEWDATA:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @_AmdAwaitTraversal(i64 4, [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA2]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define void @_cont_CallShader( -; LOWERRAYTRACINGPIPELINE-CPS-SAME: ptr [[DATA:%.*]], i32 [[TMP0:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[NEWDATA:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @_AmdAwaitShader(i64 2, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define i1 @_cont_ReportHit.struct.BuiltInTriangleIntersectionAttributes( -; LOWERRAYTRACINGPIPELINE-CPS-SAME: ptr [[DATA:%.*]], float [[T:%.*]], i32 [[HITKIND:%.*]], ptr [[TMP0:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[ORIGTPTR:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA:%.*]], ptr [[DATA]], i32 0, i32 0, i32 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[ORIGT:%.*]] = load float, ptr [[ORIGTPTR]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[ISNOHIT:%.*]] = fcmp fast uge float [[T]], [[ORIGT]] -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br i1 [[ISNOHIT]], label [[ISEND:%.*]], label [[CALLAHIT:%.*]] -; LOWERRAYTRACINGPIPELINE-CPS: callAHit: -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TRAV_DATA:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP3:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP0]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP4:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] (...) @lgc.cps.await.s_struct.AnyHitTraversalDatas(i32 3, i32 8, [[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA]], float [[T]], i32 [[HITKIND]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP3]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP4]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[DATA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_AmdRestoreSystemDataAnyHit(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: ret i1 true -; LOWERRAYTRACINGPIPELINE-CPS: isEnd: -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP6]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP8]], ptr [[TMP7]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP9]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP11]], ptr [[TMP10]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP12:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP2]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP13:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[DATA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP13]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP12]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: ret i1 false -; -; -; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define i1 @_cont_ReportHit.struct.BuiltInTriangleIntersectionAttributes2( -; LOWERRAYTRACINGPIPELINE-CPS-SAME: ptr [[DATA:%.*]], float [[T:%.*]], i32 [[HITKIND:%.*]], ptr [[TMP0:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[ORIGTPTR:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA:%.*]], ptr [[DATA]], i32 0, i32 0, i32 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[ORIGT:%.*]] = load float, ptr [[ORIGTPTR]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[ISNOHIT:%.*]] = fcmp fast uge float [[T]], [[ORIGT]] -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br i1 [[ISNOHIT]], label [[ISEND:%.*]], label [[CALLAHIT:%.*]] -; LOWERRAYTRACINGPIPELINE-CPS: callAHit: -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TRAV_DATA:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP3:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES2:%.*]], ptr [[TMP0]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP4:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] (...) @lgc.cps.await.s_struct.AnyHitTraversalDatas(i32 3, i32 8, [[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA]], float [[T]], i32 [[HITKIND]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES2]] [[TMP3]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP4]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[DATA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_AmdRestoreSystemDataAnyHit(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: ret i1 true -; LOWERRAYTRACINGPIPELINE-CPS: isEnd: -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP6]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP8]], ptr [[TMP7]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP9]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP11]], ptr [[TMP10]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP12:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP2]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP13:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[DATA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP13]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP12]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: ret i1 false -; -; -; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define i1 @_cont_ReportHit( -; LOWERRAYTRACINGPIPELINE-CPS-SAME: ptr [[DATA:%.*]], float [[T:%.*]], i32 [[HITKIND:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[ORIGTPTR:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA:%.*]], ptr [[DATA]], i32 0, i32 0, i32 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[ORIGT:%.*]] = load float, ptr [[ORIGTPTR]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[ISNOHIT:%.*]] = fcmp fast uge float [[T]], [[ORIGT]] -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br i1 [[ISNOHIT]], label [[ISEND:%.*]], label [[CALLAHIT:%.*]] -; LOWERRAYTRACINGPIPELINE-CPS: callAHit: -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TRAV_DATA:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[NEWDATA:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @_AmdAwaitAnyHit(i64 3, [[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA]], float [[T]], i32 [[HITKIND]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[DATA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP1]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_AmdRestoreSystemDataAnyHit(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: ret i1 true -; LOWERRAYTRACINGPIPELINE-CPS: isEnd: -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_AmdAcceptHitAttributes(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: ret i1 false +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP29]], !continuation.registercount [[META39]] ; ; ; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define <3 x i32> @_cont_DispatchRaysIndex3( @@ -1260,7 +1061,7 @@ attributes #5 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; ; ; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define void @MyRayGen( -; LOWERRAYTRACINGPIPELINE-CPS-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]]) #[[ATTR2:[0-9]+]] !lgc.rt.shaderstage !23 !lgc.cps !23 !continuation !35 { +; LOWERRAYTRACINGPIPELINE-CPS-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]]) #[[ATTR2:[0-9]+]] !lgc.rt.shaderstage [[META23:![0-9]+]] !lgc.cps [[META23]] !continuation [[META35:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP1:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 @@ -1276,25 +1077,66 @@ attributes #5 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP7:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP2]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP8:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP7]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP9:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[TMP8]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_TraceRay.struct.RayPayload.attr_max_32_bytes(ptr [[SYSTEM_DATA_ALLOCA]], i64 [[TMP9]], i32 16, i32 -1, i32 0, i32 1, i32 0, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0x3F50624DE0000000, float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 1.000000e+04, ptr [[TMP4]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP10:%.*]] = load <4 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA36]] -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP11:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[EXTRACT:%.*]] = extractelement <3 x i32> [[TMP11]], i8 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP12:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[EXTRACT1:%.*]] = extractelement <3 x i32> [[TMP12]], i8 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP13:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP3]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP14:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP13]], [[DX_TYPES_RESOURCEPROPERTIES]] { i32 4098, i32 1033 }) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP15:%.*]] = extractelement <4 x float> [[TMP10]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP16:%.*]] = extractelement <4 x float> [[TMP10]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP17:%.*]] = extractelement <4 x float> [[TMP10]], i64 2 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP18:%.*]] = extractelement <4 x float> [[TMP10]], i64 3 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @dx.op.textureStore.f32(i32 67, [[DX_TYPES_HANDLE]] [[TMP14]], i32 [[EXTRACT]], i32 [[EXTRACT1]], i32 undef, float [[TMP15]], float [[TMP16]], float [[TMP17]], float [[TMP18]], i8 15) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[DIS_DATA_I:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[SYS_DATA_I:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I]], 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TRAV_DATA_I:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA_I]], 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[ADDR_I:%.*]] = call i64 @_AmdGetResumePointAddr() #[[ATTR3:[0-9]+]] +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TRAV_DATA2_I:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]], i64 [[ADDR_I]], 5 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP4]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[TMP10]], i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[TMP11]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP13]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[TMP10]], i32 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[TMP14]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP16]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP17:%.*]] = getelementptr i32, ptr [[TMP14]], i64 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP18]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[TMP14]], i64 2 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP19]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP20]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP21:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] (...) @lgc.cps.await.s_struct.DispatchSystemDatas(i32 4, i32 4, [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA2_I]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_RAYPAYLOAD]] poison, ptr [[TMP4]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP22:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP4]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP23:%.*]] = getelementptr i32, ptr [[TMP22]], i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP24:%.*]] = getelementptr i32, ptr [[TMP23]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP25:%.*]] = load i32, ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP25]], ptr [[TMP24]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP26:%.*]] = getelementptr i32, ptr [[TMP22]], i32 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP27:%.*]] = getelementptr i32, ptr [[TMP26]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP28:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP28]], ptr [[TMP27]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP29:%.*]] = getelementptr i32, ptr [[TMP26]], i64 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP30:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP30]], ptr [[TMP29]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP31:%.*]] = getelementptr i32, ptr [[TMP26]], i64 2 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP32:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP32]], ptr [[TMP31]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP21]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX1:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[SYSTEM_DATA_ALLOCA]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX1]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br label [[DOTSPLIT:%.*]] +; LOWERRAYTRACINGPIPELINE-CPS: .split: +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP33:%.*]] = load <4 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA36]] +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP34:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[EXTRACT:%.*]] = extractelement <3 x i32> [[TMP34]], i8 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP35:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[EXTRACT1:%.*]] = extractelement <3 x i32> [[TMP35]], i8 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP36:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP3]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP37:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP36]], [[DX_TYPES_RESOURCEPROPERTIES]] { i32 4098, i32 1033 }) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP38:%.*]] = extractelement <4 x float> [[TMP33]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP39:%.*]] = extractelement <4 x float> [[TMP33]], i64 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP40:%.*]] = extractelement <4 x float> [[TMP33]], i64 2 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP41:%.*]] = extractelement <4 x float> [[TMP33]], i64 3 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @dx.op.textureStore.f32(i32 67, [[DX_TYPES_HANDLE]] [[TMP37]], i32 [[EXTRACT]], i32 [[EXTRACT1]], i32 undef, float [[TMP38]], float [[TMP39]], float [[TMP40]], float [[TMP41]], i8 15) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @llvm.lifetime.end.p0(i64 16, ptr [[TMP5]]) #[[ATTR1]] ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: ret void ; ; ; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define void @MyClosestHitShader( -; LOWERRAYTRACINGPIPELINE-CPS-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage !39 !lgc.cps !40 !continuation !41 { +; LOWERRAYTRACINGPIPELINE-CPS-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage [[META39:![0-9]+]] !lgc.cps [[META40:![0-9]+]] !continuation [[META41:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_RAYPAYLOAD:%.*]], align 8 @@ -1349,7 +1191,7 @@ attributes #5 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; ; ; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define void @MyAnyHitShader( -; LOWERRAYTRACINGPIPELINE-CPS-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]], [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] [[TMP1:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage !42 !lgc.cps !39 !continuation !43 { +; LOWERRAYTRACINGPIPELINE-CPS-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]], [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] [[TMP1:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage [[META42:![0-9]+]] !lgc.cps [[META39]] !continuation [[META43:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_HITDATA:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP4:%.*]] = alloca [[STRUCT_HITDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP5:%.*]] = alloca [[STRUCT_HITDATA]], align 8 @@ -1385,261 +1227,333 @@ attributes #5 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP25:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP26:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_HITDATA]] [[TMP26]], ptr [[TMP4]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP27:%.*]] = call <3 x float> @_cont_ObjectRayOrigin3(ptr [[TMP25]], ptr [[TMP4]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[EXTRACT1:%.*]] = extractelement <3 x float> [[TMP27]], i8 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP28:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP29:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_HITDATA]] [[TMP29]], ptr [[TMP5]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP30:%.*]] = call <3 x float> @_cont_ObjectRayDirection3(ptr [[TMP28]], ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[EXTRACT:%.*]] = extractelement <3 x float> [[TMP30]], i8 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP31:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP32:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_HITDATA]] [[TMP32]], ptr [[TMP3]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP33:%.*]] = call float @_cont_RayTCurrent(ptr [[TMP31]], ptr [[TMP3]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP34:%.*]] = fmul fast float [[TMP33]], [[EXTRACT]] -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP35:%.*]] = fadd fast float [[TMP34]], [[EXTRACT1]] -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP36:%.*]] = fcmp fast ogt float [[TMP35]], 0.000000e+00 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP37:%.*]] = fcmp fast ogt float [[TMP35]], 1.000000e+00 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP38:%.*]] = fcmp fast ogt float [[TMP35]], -1.000000e+00 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br i1 [[TMP36]], label [[TMP39:%.*]], label [[TMP84:%.*]] -; LOWERRAYTRACINGPIPELINE-CPS: 39: +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[RES_1_I1:%.*]] = load float, ptr [[TMP4]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[RESPTR_2_I2:%.*]] = getelementptr [[STRUCT_HITDATA]], ptr [[TMP4]], i32 0, i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[RES_2_I3:%.*]] = load float, ptr [[RESPTR_2_I2]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[RESPTR_3_I4:%.*]] = getelementptr [[STRUCT_HITDATA]], ptr [[TMP4]], i32 0, i32 0, i32 2 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[RES_3_I5:%.*]] = load float, ptr [[RESPTR_3_I4]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[VAL_0_I6:%.*]] = insertelement <3 x float> undef, float [[RES_1_I1]], i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[VAL_1_I7:%.*]] = insertelement <3 x float> [[VAL_0_I6]], float [[RES_2_I3]], i32 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[VAL_2_I8:%.*]] = insertelement <3 x float> [[VAL_1_I7]], float [[RES_3_I5]], i32 2 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[EXTRACT1:%.*]] = extractelement <3 x float> [[VAL_2_I8]], i8 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP27:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP28:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_HITDATA]] [[TMP28]], ptr [[TMP5]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[RESPTR_1_I:%.*]] = getelementptr [[STRUCT_HITDATA]], ptr [[TMP5]], i32 0, i32 1, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[RES_1_I:%.*]] = load float, ptr [[RESPTR_1_I]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[RESPTR_2_I:%.*]] = getelementptr [[STRUCT_HITDATA]], ptr [[TMP5]], i32 0, i32 1, i32 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[RES_2_I:%.*]] = load float, ptr [[RESPTR_2_I]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[RESPTR_3_I:%.*]] = getelementptr [[STRUCT_HITDATA]], ptr [[TMP5]], i32 0, i32 1, i32 2 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[RES_3_I:%.*]] = load float, ptr [[RESPTR_3_I]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[VAL_0_I:%.*]] = insertelement <3 x float> undef, float [[RES_1_I]], i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[VAL_1_I:%.*]] = insertelement <3 x float> [[VAL_0_I]], float [[RES_2_I]], i32 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[VAL_2_I:%.*]] = insertelement <3 x float> [[VAL_1_I]], float [[RES_3_I]], i32 2 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[EXTRACT:%.*]] = extractelement <3 x float> [[VAL_2_I]], i8 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP29:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP30:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_HITDATA]] [[TMP30]], ptr [[TMP3]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[RESPTR_I:%.*]] = getelementptr [[STRUCT_HITDATA]], ptr [[TMP3]], i32 0, i32 2 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[RES_I:%.*]] = load float, ptr [[RESPTR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP31:%.*]] = fmul fast float [[RES_I]], [[EXTRACT]] +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP32:%.*]] = fadd fast float [[TMP31]], [[EXTRACT1]] +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP33:%.*]] = fcmp fast ogt float [[TMP32]], 0.000000e+00 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP34:%.*]] = fcmp fast ogt float [[TMP32]], 1.000000e+00 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP35:%.*]] = fcmp fast ogt float [[TMP32]], -1.000000e+00 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br i1 [[TMP33]], label [[TMP36:%.*]], label [[TMP81:%.*]] +; LOWERRAYTRACINGPIPELINE-CPS: 36: ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store <4 x float> [[TMP24]], ptr [[TMP23]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br i1 [[TMP37]], label [[TMP40:%.*]], label [[TMP62:%.*]] -; LOWERRAYTRACINGPIPELINE-CPS: 40: -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP41:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_AcceptHitAndEndSearch(ptr [[TMP41]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br i1 [[TMP34]], label [[TMP37:%.*]], label [[TMP59:%.*]] +; LOWERRAYTRACINGPIPELINE-CPS: 37: +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP38:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_AcceptHitAndEndSearch(ptr [[TMP38]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP42:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP43:%.*]] = getelementptr i32, ptr [[TMP42]], i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP39:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP40:%.*]] = getelementptr i32, ptr [[TMP39]], i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP41:%.*]] = getelementptr i32, ptr [[TMP40]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP42:%.*]] = load i32, ptr [[TMP41]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP42]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP43:%.*]] = getelementptr i32, ptr [[TMP39]], i32 1 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP44:%.*]] = getelementptr i32, ptr [[TMP43]], i64 0 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP45:%.*]] = load i32, ptr [[TMP44]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP45]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP46:%.*]] = getelementptr i32, ptr [[TMP42]], i32 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP47:%.*]] = getelementptr i32, ptr [[TMP46]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP48:%.*]] = load i32, ptr [[TMP47]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP48]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP49:%.*]] = getelementptr i32, ptr [[TMP46]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP50:%.*]] = load i32, ptr [[TMP49]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP50]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP51:%.*]] = getelementptr i32, ptr [[TMP46]], i64 2 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP52:%.*]] = load i32, ptr [[TMP51]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP52]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP53:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP54:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP45]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP46:%.*]] = getelementptr i32, ptr [[TMP43]], i64 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP47:%.*]] = load i32, ptr [[TMP46]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP47]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP48:%.*]] = getelementptr i32, ptr [[TMP43]], i64 2 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP49:%.*]] = load i32, ptr [[TMP48]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP49]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP50:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP51:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP52:%.*]] = load i32, ptr [[TMP50]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP52]], ptr [[TMP51]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP53:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP54:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 1 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP55:%.*]] = load i32, ptr [[TMP53]], align 4 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP55]], ptr [[TMP54]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP56:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP57:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP58:%.*]] = load i32, ptr [[TMP56]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP58]], ptr [[TMP57]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP59:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP10]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP60:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP60]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP59]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP61:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 18, {} poison, [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP61]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP56:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP10]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP57:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP57]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP56]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP58:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 18, {} poison, [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP58]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: unreachable -; LOWERRAYTRACINGPIPELINE-CPS: 62: -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP63:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_AcceptHitAndEndSearch(ptr [[TMP63]]) +; LOWERRAYTRACINGPIPELINE-CPS: 59: +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP60:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_AcceptHitAndEndSearch(ptr [[TMP60]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP64:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP65:%.*]] = getelementptr i32, ptr [[TMP64]], i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP61:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP62:%.*]] = getelementptr i32, ptr [[TMP61]], i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP63:%.*]] = getelementptr i32, ptr [[TMP62]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP64:%.*]] = load i32, ptr [[TMP63]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP64]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP65:%.*]] = getelementptr i32, ptr [[TMP61]], i32 1 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP66:%.*]] = getelementptr i32, ptr [[TMP65]], i64 0 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP67:%.*]] = load i32, ptr [[TMP66]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP67]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP68:%.*]] = getelementptr i32, ptr [[TMP64]], i32 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP69:%.*]] = getelementptr i32, ptr [[TMP68]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP70:%.*]] = load i32, ptr [[TMP69]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP70]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP71:%.*]] = getelementptr i32, ptr [[TMP68]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP72:%.*]] = load i32, ptr [[TMP71]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP72]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP73:%.*]] = getelementptr i32, ptr [[TMP68]], i64 2 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP74:%.*]] = load i32, ptr [[TMP73]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP74]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP75:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP76:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP67]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP68:%.*]] = getelementptr i32, ptr [[TMP65]], i64 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP69:%.*]] = load i32, ptr [[TMP68]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP69]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP70:%.*]] = getelementptr i32, ptr [[TMP65]], i64 2 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP71:%.*]] = load i32, ptr [[TMP70]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP71]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP72:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP73:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP74:%.*]] = load i32, ptr [[TMP72]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP74]], ptr [[TMP73]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP75:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP76:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i64 1 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP77:%.*]] = load i32, ptr [[TMP75]], align 4 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP77]], ptr [[TMP76]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP78:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP79:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP80:%.*]] = load i32, ptr [[TMP78]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP80]], ptr [[TMP79]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP81:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP9]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP82:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP82]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP81]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP83:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 18, {} poison, [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP83]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP78:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP9]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP79:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP79]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP78]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP80:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 18, {} poison, [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP80]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: unreachable -; LOWERRAYTRACINGPIPELINE-CPS: 84: -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br i1 [[TMP38]], label [[TMP85:%.*]], label [[TMP130:%.*]] -; LOWERRAYTRACINGPIPELINE-CPS: 85: -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br i1 [[TMP37]], label [[TMP86:%.*]], label [[TMP108:%.*]] -; LOWERRAYTRACINGPIPELINE-CPS: 86: -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP87:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_IgnoreHit(ptr [[TMP87]]) +; LOWERRAYTRACINGPIPELINE-CPS: 81: +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br i1 [[TMP35]], label [[TMP82:%.*]], label [[TMP127:%.*]] +; LOWERRAYTRACINGPIPELINE-CPS: 82: +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br i1 [[TMP34]], label [[TMP83:%.*]], label [[TMP105:%.*]] +; LOWERRAYTRACINGPIPELINE-CPS: 83: +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP84:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_IgnoreHit(ptr [[TMP84]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP88:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP89:%.*]] = getelementptr i32, ptr [[TMP88]], i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP85:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP86:%.*]] = getelementptr i32, ptr [[TMP85]], i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP87:%.*]] = getelementptr i32, ptr [[TMP86]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP88:%.*]] = load i32, ptr [[TMP87]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP88]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP89:%.*]] = getelementptr i32, ptr [[TMP85]], i32 1 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP90:%.*]] = getelementptr i32, ptr [[TMP89]], i64 0 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP91:%.*]] = load i32, ptr [[TMP90]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP91]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP92:%.*]] = getelementptr i32, ptr [[TMP88]], i32 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP93:%.*]] = getelementptr i32, ptr [[TMP92]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP94:%.*]] = load i32, ptr [[TMP93]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP94]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP95:%.*]] = getelementptr i32, ptr [[TMP92]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP96:%.*]] = load i32, ptr [[TMP95]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP96]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP97:%.*]] = getelementptr i32, ptr [[TMP92]], i64 2 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP98:%.*]] = load i32, ptr [[TMP97]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP98]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP99:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP100:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP91]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP92:%.*]] = getelementptr i32, ptr [[TMP89]], i64 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP93:%.*]] = load i32, ptr [[TMP92]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP93]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP94:%.*]] = getelementptr i32, ptr [[TMP89]], i64 2 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP95:%.*]] = load i32, ptr [[TMP94]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP95]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP96:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP97:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP98:%.*]] = load i32, ptr [[TMP96]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP98]], ptr [[TMP97]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP99:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP100:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 1 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP101:%.*]] = load i32, ptr [[TMP99]], align 4 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP101]], ptr [[TMP100]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP102:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP103:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP104:%.*]] = load i32, ptr [[TMP102]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP104]], ptr [[TMP103]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP105:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP8]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP106:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP106]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP105]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP107:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 18, {} poison, [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP107]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP102:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP8]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP103:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP103]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP102]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP104:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 18, {} poison, [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP104]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: unreachable -; LOWERRAYTRACINGPIPELINE-CPS: 108: -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP109:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_IgnoreHit(ptr [[TMP109]]) +; LOWERRAYTRACINGPIPELINE-CPS: 105: +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP106:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_IgnoreHit(ptr [[TMP106]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP110:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP111:%.*]] = getelementptr i32, ptr [[TMP110]], i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP107:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP108:%.*]] = getelementptr i32, ptr [[TMP107]], i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP109:%.*]] = getelementptr i32, ptr [[TMP108]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP110:%.*]] = load i32, ptr [[TMP109]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP110]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP111:%.*]] = getelementptr i32, ptr [[TMP107]], i32 1 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP112:%.*]] = getelementptr i32, ptr [[TMP111]], i64 0 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP113:%.*]] = load i32, ptr [[TMP112]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP113]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP114:%.*]] = getelementptr i32, ptr [[TMP110]], i32 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP115:%.*]] = getelementptr i32, ptr [[TMP114]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP116:%.*]] = load i32, ptr [[TMP115]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP116]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP117:%.*]] = getelementptr i32, ptr [[TMP114]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP118:%.*]] = load i32, ptr [[TMP117]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP118]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP119:%.*]] = getelementptr i32, ptr [[TMP114]], i64 2 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP120:%.*]] = load i32, ptr [[TMP119]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP120]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP121:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP122:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP113]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP114:%.*]] = getelementptr i32, ptr [[TMP111]], i64 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP115:%.*]] = load i32, ptr [[TMP114]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP115]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP116:%.*]] = getelementptr i32, ptr [[TMP111]], i64 2 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP117:%.*]] = load i32, ptr [[TMP116]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP117]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_1_ANYHIT_IN]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP118:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP119:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP120:%.*]] = load i32, ptr [[TMP118]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP120]], ptr [[TMP119]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP121:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP122:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 1 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP123:%.*]] = load i32, ptr [[TMP121]], align 4 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP123]], ptr [[TMP122]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP124:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP125:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP126:%.*]] = load i32, ptr [[TMP124]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP126]], ptr [[TMP125]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP127:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP7]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP128:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP128]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP127]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP129:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 18, {} poison, [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP129]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP124:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP7]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP125:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP125]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP124]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP126:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 18, {} poison, [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP126]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: unreachable -; LOWERRAYTRACINGPIPELINE-CPS: 130: +; LOWERRAYTRACINGPIPELINE-CPS: 127: ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store <4 x float> [[TMP24]], ptr [[TMP23]], align 4 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_AcceptHit(ptr [[SYSTEM_DATA_ALLOCA]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP131:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP132:%.*]] = getelementptr i32, ptr [[TMP131]], i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP128:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP12]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP129:%.*]] = getelementptr i32, ptr [[TMP128]], i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP130:%.*]] = getelementptr i32, ptr [[TMP129]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP131:%.*]] = load i32, ptr [[TMP130]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP131]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP132:%.*]] = getelementptr i32, ptr [[TMP128]], i32 1 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP133:%.*]] = getelementptr i32, ptr [[TMP132]], i64 0 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP134:%.*]] = load i32, ptr [[TMP133]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP134]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP135:%.*]] = getelementptr i32, ptr [[TMP131]], i32 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP136:%.*]] = getelementptr i32, ptr [[TMP135]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP137:%.*]] = load i32, ptr [[TMP136]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP137]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP138:%.*]] = getelementptr i32, ptr [[TMP135]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP139:%.*]] = load i32, ptr [[TMP138]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP139]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP140:%.*]] = getelementptr i32, ptr [[TMP135]], i64 2 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP141:%.*]] = load i32, ptr [[TMP140]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP141]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP142:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP143:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP134]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP135:%.*]] = getelementptr i32, ptr [[TMP132]], i64 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP136:%.*]] = load i32, ptr [[TMP135]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP136]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP137:%.*]] = getelementptr i32, ptr [[TMP132]], i64 2 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP138:%.*]] = load i32, ptr [[TMP137]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP138]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP139:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP140:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP141:%.*]] = load i32, ptr [[TMP139]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP141]], ptr [[TMP140]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP142:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP143:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 1 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP144:%.*]] = load i32, ptr [[TMP142]], align 4 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP144]], ptr [[TMP143]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP145:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP146:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 1 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP147:%.*]] = load i32, ptr [[TMP145]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP147]], ptr [[TMP146]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP148:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP6]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP149:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP149]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP148]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP150:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 18, {} poison, [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP150]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP145:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP6]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP146:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP146]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP145]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP147:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 18, {} poison, [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP147]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: unreachable ; ; ; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define void @MyIntersectionShader( -; LOWERRAYTRACINGPIPELINE-CPS-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]], [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage !40 !lgc.cps !44 !continuation !45 { +; LOWERRAYTRACINGPIPELINE-CPS-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]], [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage [[META40]] !lgc.cps [[META44:![0-9]+]] !continuation [[META45:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_HITDATA:%.*]], align 8 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP4:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], align 4 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_ANYHITTRAVERSALDATA]], align 8 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP4:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @continuations.getSystemData.s_struct.AnyHitTraversalDatas() -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP4]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP5:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @continuations.getSystemData.s_struct.AnyHitTraversalDatas() +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP5]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP7:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_HITDATA]] [[TMP7]], ptr [[TMP2]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP8:%.*]] = call float @_cont_RayTCurrent(ptr [[TMP6]], ptr [[TMP2]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP9:%.*]] = bitcast ptr [[TMP3]] to ptr +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP6]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP8:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_HITDATA]] [[TMP8]], ptr [[TMP2]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[RESPTR_I:%.*]] = getelementptr [[STRUCT_HITDATA]], ptr [[TMP2]], i32 0, i32 2 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[RES_I:%.*]] = load float, ptr [[RESPTR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP9:%.*]] = bitcast ptr [[TMP4]] to ptr ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[TMP9]]) #[[ATTR1]] -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP10:%.*]] = call i1 @_cont_ReportHit.struct.BuiltInTriangleIntersectionAttributes(ptr [[SYSTEM_DATA_ALLOCA]], float [[TMP8]], i32 0, ptr [[TMP3]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP12:%.*]] = call i1 @_cont_IsEndSearch(ptr [[TMP11]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br i1 [[TMP12]], label [[TMP13:%.*]], label [[TMP15:%.*]] -; LOWERRAYTRACINGPIPELINE-CPS: 13: -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP14:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 10, {} poison, [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP14]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[ORIGTPTR_I:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[ORIGT_I:%.*]] = load float, ptr [[ORIGTPTR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[ISNOHIT_I:%.*]] = fcmp fast uge float [[RES_I]], [[ORIGT_I]] +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br i1 [[ISNOHIT_I]], label [[ISEND_I:%.*]], label [[CALLAHIT_I:%.*]] +; LOWERRAYTRACINGPIPELINE-CPS: callAHit.i: +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TRAV_DATA_I:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP10:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP4]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP11:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] (...) @lgc.cps.await.s_struct.AnyHitTraversalDatas(i32 3, i32 8, [[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA_I]], float [[RES_I]], i32 0, [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP10]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP11]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX1:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP12]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX1]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br label [[_CONT_REPORTHIT_EXIT:%.*]] +; LOWERRAYTRACINGPIPELINE-CPS: isEnd.i: +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP13]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP15]], ptr [[TMP14]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP16]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP18]], ptr [[TMP17]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP19:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP3]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP20:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP20]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP19]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br label [[_CONT_REPORTHIT_EXIT]] +; LOWERRAYTRACINGPIPELINE-CPS: _cont_ReportHit.exit: +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP22:%.*]] = call i1 @_cont_IsEndSearch(ptr [[TMP21]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br i1 [[TMP22]], label [[TMP23:%.*]], label [[TMP25:%.*]] +; LOWERRAYTRACINGPIPELINE-CPS: 23: +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP24:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 10, {} poison, [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP24]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: unreachable -; LOWERRAYTRACINGPIPELINE-CPS: 15: +; LOWERRAYTRACINGPIPELINE-CPS: 25: ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[TMP9]]) #[[ATTR1]] -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP16:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 10, {} poison, [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP16]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP26:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 10, {} poison, [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP26]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: unreachable ; ; ; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define void @MyIntersectionShader2( -; LOWERRAYTRACINGPIPELINE-CPS-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]], [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage !40 !lgc.cps !44 !continuation !46 { +; LOWERRAYTRACINGPIPELINE-CPS-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]], [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage [[META40]] !lgc.cps [[META44]] !continuation [[META46:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_HITDATA:%.*]], align 8 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES2:%.*]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP4:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES2:%.*]], align 4 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_ANYHITTRAVERSALDATA]], align 8 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP4:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @continuations.getSystemData.s_struct.AnyHitTraversalDatas() -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP4]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP5:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] @continuations.getSystemData.s_struct.AnyHitTraversalDatas() +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP5]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP7:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_HITDATA]] [[TMP7]], ptr [[TMP2]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP8:%.*]] = call float @_cont_RayTCurrent(ptr [[TMP6]], ptr [[TMP2]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP9:%.*]] = bitcast ptr [[TMP3]] to ptr +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP6]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP8:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_HITDATA]] [[TMP8]], ptr [[TMP2]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[RESPTR_I:%.*]] = getelementptr [[STRUCT_HITDATA]], ptr [[TMP2]], i32 0, i32 2 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[RES_I:%.*]] = load float, ptr [[RESPTR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP9:%.*]] = bitcast ptr [[TMP4]] to ptr ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[TMP9]]) #[[ATTR1]] -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP10:%.*]] = call i1 @_cont_ReportHit.struct.BuiltInTriangleIntersectionAttributes2(ptr [[SYSTEM_DATA_ALLOCA]], float [[TMP8]], i32 0, ptr [[TMP3]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP12:%.*]] = call i1 @_cont_IsEndSearch(ptr [[TMP11]]) -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br i1 [[TMP12]], label [[TMP13:%.*]], label [[TMP15:%.*]] -; LOWERRAYTRACINGPIPELINE-CPS: 13: -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP14:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 10, {} poison, [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP14]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[ORIGTPTR_I:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[ORIGT_I:%.*]] = load float, ptr [[ORIGTPTR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[ISNOHIT_I:%.*]] = fcmp fast uge float [[RES_I]], [[ORIGT_I]] +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br i1 [[ISNOHIT_I]], label [[ISEND_I:%.*]], label [[CALLAHIT_I:%.*]] +; LOWERRAYTRACINGPIPELINE-CPS: callAHit.i: +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TRAV_DATA_I:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP10:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES2]], ptr [[TMP4]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP11:%.*]] = call [[STRUCT_ANYHITTRAVERSALDATA]] (...) @lgc.cps.await.s_struct.AnyHitTraversalDatas(i32 3, i32 8, [[STRUCT_ANYHITTRAVERSALDATA]] [[TRAV_DATA_I]], float [[RES_I]], i32 0, [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES2]] [[TMP10]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP11]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[LOCAL_ROOT_INDEX1:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP12]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX1]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br label [[_CONT_REPORTHIT_EXIT:%.*]] +; LOWERRAYTRACINGPIPELINE-CPS: isEnd.i: +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP13]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP15]], ptr [[TMP14]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 1 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP16]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: store i32 [[TMP18]], ptr [[TMP17]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP19:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP3]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP20:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP20]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP19]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br label [[_CONT_REPORTHIT_EXIT]] +; LOWERRAYTRACINGPIPELINE-CPS: _cont_ReportHit.exit: +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP22:%.*]] = call i1 @_cont_IsEndSearch(ptr [[TMP21]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: br i1 [[TMP22]], label [[TMP23:%.*]], label [[TMP25:%.*]] +; LOWERRAYTRACINGPIPELINE-CPS: 23: +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP24:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 10, {} poison, [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP24]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: unreachable -; LOWERRAYTRACINGPIPELINE-CPS: 15: +; LOWERRAYTRACINGPIPELINE-CPS: 25: ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[TMP9]]) #[[ATTR1]] -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP16:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 10, {} poison, [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP16]]) +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP26:%.*]] = load [[STRUCT_ANYHITTRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-CPS-NEXT: call void (...) @lgc.cps.jump(i32 [[RETURN_ADDR]], i32 10, {} poison, [[STRUCT_ANYHITTRAVERSALDATA]] [[TMP26]]) ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: unreachable ; ; ; LOWERRAYTRACINGPIPELINE-CPS-LABEL: define void @MyMissShader( -; LOWERRAYTRACINGPIPELINE-CPS-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage !44 !lgc.cps !40 !continuation !47 { +; LOWERRAYTRACINGPIPELINE-CPS-SAME: {} [[CONT_STATE:%.*]], i32 [[RETURN_ADDR:%.*]], i32 [[SHADER_INDEX:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage [[META44]] !lgc.cps [[META40]] !continuation [[META47:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_RAYPAYLOAD:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-CPS-NEXT: [[TMP3:%.*]] = call [[STRUCT_SYSTEMDATA]] @continuations.getSystemData.s_struct.SystemDatas() diff --git a/shared/continuations/test/dx/payload-caller-in-paq.ll b/shared/continuations/test/dx/payload-caller-in-paq.ll index 3313fdf8c4..626217e56b 100644 --- a/shared/continuations/test/dx/payload-caller-in-paq.ll +++ b/shared/continuations/test/dx/payload-caller-in-paq.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function RayGen --version 3 -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,remove-types-metadata' -S %s 2>%t.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,remove-types-metadata' -S %s 2> %t.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s ; RUN: count 0 < %t.stderr ; Test the special case of payload import in the caller after TraceRay. Here, we cast the @@ -30,8 +30,8 @@ target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32- ; Function Attrs: nounwind define void @RayGen() #0 { -; LOWERRAYTRACINGPIPELINE-LABEL: define void @RayGen -; LOWERRAYTRACINGPIPELINE-SAME: () #[[ATTR0:[0-9]+]] !lgc.rt.shaderstage !24 !continuation.entry !14 !continuation.registercount !24 !continuation !27 { +; LOWERRAYTRACINGPIPELINE-LABEL: define void @RayGen( +; LOWERRAYTRACINGPIPELINE-SAME: ) #[[ATTR0:[0-9]+]] !lgc.rt.shaderstage [[META24:![0-9]+]] !continuation.entry [[META14:![0-9]+]] !continuation.registercount [[META24]] !continuation [[META27:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP1:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 @@ -49,41 +49,46 @@ define void @RayGen() #0 { ; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA_I:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA_I:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I]], 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA_I:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA_I]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = load float, ptr [[TMP4]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP10]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = call ptr inttoptr (i64 4 to ptr)([[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]]), !continuation.registercount !32, !continuation.returnedRegistercount !33 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @await.struct.DispatchSystemData(ptr [[TMP11]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_MYPAYLOAD]], ptr [[TMP4]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = load float, ptr [[TMP10]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP11]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = call ptr inttoptr (i64 4 to ptr)([[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]]), !continuation.registercount [[META32:![0-9]+]], !continuation.returnedRegistercount !33 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @await.struct.DispatchSystemData(ptr [[TMP12]]) ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_MYPAYLOAD]] poison, ptr [[TMP4]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = load float, ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP13]], ptr [[TMP4]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = getelementptr inbounds [[STRUCT_MYPAYLOAD]], ptr [[TMP4]], i32 0, i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_MYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 1), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP15]], ptr [[TMP14]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = getelementptr inbounds [[STRUCT_MYPAYLOAD]], ptr [[TMP4]], i32 0, i32 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_MYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 1), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = getelementptr inbounds [[STRUCT_MYPAYLOAD]], ptr [[TMP4]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = load float, ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP15]], ptr [[TMP14]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = getelementptr inbounds [[STRUCT_MYPAYLOAD]], ptr [[TMP4]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_MYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 1), align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP17]], ptr [[TMP16]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[TMP16]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP19:%.*]] = load i32, ptr getelementptr ([[STRUCT_MYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 2), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP19]], ptr [[TMP18]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP12]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = getelementptr inbounds [[STRUCT_MYPAYLOAD]], ptr [[TMP4]], i32 0, i32 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[TMP18]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[TMP19]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP21:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_MYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 1), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP21]], ptr [[TMP20]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[TMP19]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP23:%.*]] = load i32, ptr getelementptr ([[STRUCT_MYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 2), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP23]], ptr [[TMP22]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP13]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP20:%.*]] = load float, ptr [[TMP6]], align 8, !tbaa [[TBAA28]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_MYPAYLOAD]], ptr [[TMP4]], i32 0, i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4, !tbaa [[TBAA34:![0-9]+]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP23:%.*]] = sitofp i32 [[TMP22]] to float -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP24:%.*]] = getelementptr inbounds [[STRUCT_MYPAYLOAD]], ptr [[TMP4]], i32 0, i32 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP25:%.*]] = load double, ptr [[TMP24]], align 8, !tbaa [[TBAA36:![0-9]+]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP26:%.*]] = fptrunc double [[TMP25]] to float -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP27:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() -; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT:%.*]] = extractelement <3 x i32> [[TMP27]], i8 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() -; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT1:%.*]] = extractelement <3 x i32> [[TMP28]], i8 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP3]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP30:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP29]], [[DX_TYPES_RESOURCEPROPERTIES]] { i32 4098, i32 1033 }) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @dx.op.textureStore.f32(i32 67, [[DX_TYPES_HANDLE]] [[TMP30]], i32 [[EXTRACT]], i32 [[EXTRACT1]], i32 undef, float [[TMP20]], float [[TMP23]], float [[TMP26]], float 0.000000e+00, i8 15) +; LOWERRAYTRACINGPIPELINE-NEXT: br label [[DOTSPLIT:%.*]] +; LOWERRAYTRACINGPIPELINE: .split: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP24:%.*]] = load float, ptr [[TMP6]], align 8, !tbaa [[TBAA28]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP25:%.*]] = getelementptr inbounds [[STRUCT_MYPAYLOAD]], ptr [[TMP4]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP25]], align 4, !tbaa [[TBAA34:![0-9]+]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP27:%.*]] = sitofp i32 [[TMP26]] to float +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = getelementptr inbounds [[STRUCT_MYPAYLOAD]], ptr [[TMP4]], i32 0, i32 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = load double, ptr [[TMP28]], align 8, !tbaa [[TBAA36:![0-9]+]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP30:%.*]] = fptrunc double [[TMP29]] to float +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP31:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() +; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT:%.*]] = extractelement <3 x i32> [[TMP31]], i8 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP32:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() +; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT1:%.*]] = extractelement <3 x i32> [[TMP32]], i8 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP33:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP3]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP33]], [[DX_TYPES_RESOURCEPROPERTIES]] { i32 4098, i32 1033 }) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @dx.op.textureStore.f32(i32 67, [[DX_TYPES_HANDLE]] [[TMP34]], i32 [[EXTRACT]], i32 [[EXTRACT1]], i32 undef, float [[TMP24]], float [[TMP27]], float [[TMP30]], float 0.000000e+00, i8 15) ; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.end.p0(i64 16, ptr [[TMP5]]) #[[ATTR0]] -; LOWERRAYTRACINGPIPELINE-NEXT: ret void +; LOWERRAYTRACINGPIPELINE-NEXT: ret void, !continuation.registercount [[META25:![0-9]+]] ; %1 = load %dx.types.Handle, %dx.types.Handle* @"\01?myAccelerationStructure@@3URaytracingAccelerationStructure@@A", align 4 %2 = load %dx.types.Handle, %dx.types.Handle* @"\01?gOutput@@3V?$RWTexture2D@V?$vector@M$03@@@@A", align 4 diff --git a/shared/continuations/test/dx/payload-save-registers.ll b/shared/continuations/test/dx/payload-save-registers.ll index fafd2d536b..efe5b41f12 100644 --- a/shared/continuations/test/dx/payload-save-registers.ll +++ b/shared/continuations/test/dx/payload-save-registers.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2>%t.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2> %t.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s ; RUN: count 0 < %t.stderr ; Test that we correctly save and restore registers before/after recursive @@ -30,14 +30,13 @@ target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32- ; Function Attrs: nounwind define void @Miss(%struct.OuterPayload* noalias nocapture %outerPayload) #0 !types !23 { ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.DispatchSystemData @Miss( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] !lgc.rt.shaderstage !26 !continuation.registercount !24 !continuation !27 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] !lgc.rt.shaderstage [[META26:![0-9]+]] !continuation.registercount [[META24:![0-9]+]] !continuation [[META27:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_OUTERPAYLOAD:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = call [[STRUCT_SYSTEMDATA]] @continuations.getSystemData.s_struct.SystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_SYSTEMDATA]] [[TMP3]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP4]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_OUTERPAYLOAD]], ptr [[TMP2]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[TMP5]], i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i64 0 @@ -107,9 +106,24 @@ define void @Miss(%struct.OuterPayload* noalias nocapture %outerPayload) #0 !typ ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP53:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP52]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP54:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[TMP53]]) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP55:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_TraceRay.struct.InnerPayload.attr_max_32_bytes(ptr [[TMP55]], i64 [[TMP54]], i32 0, i32 0, i32 0, i32 0, i32 0, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 1.000000e+00, ptr [[TMP47]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP56:%.*]] = load float, ptr [[TMP51]], align 4, !tbaa [[TBAA28]] -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP56]], ptr [[TMP49]], align 4, !tbaa [[TBAA28]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA_I:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP55]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA_I:%.*]] = insertvalue [[STRUCT_SYSTEMDATA]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I]], 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA_I:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA_I]], 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP56:%.*]] = getelementptr inbounds [[STRUCT_INNERPAYLOAD]], ptr [[TMP47]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP57:%.*]] = load float, ptr [[TMP56]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP57]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP58:%.*]] = call ptr inttoptr (i64 4 to ptr)([[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]]), !continuation.registercount [[META32:![0-9]+]], !continuation.returnedRegistercount !32 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP59:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @await.struct.DispatchSystemData(ptr [[TMP58]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_INNERPAYLOAD]] poison, ptr [[TMP47]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP60:%.*]] = getelementptr inbounds [[STRUCT_INNERPAYLOAD]], ptr [[TMP47]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP61:%.*]] = load float, ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP61]], ptr [[TMP60]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP59]], ptr [[TMP55]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; LOWERRAYTRACINGPIPELINE-NEXT: br label [[DOTSPLIT:%.*]] +; LOWERRAYTRACINGPIPELINE: .split: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP62:%.*]] = load float, ptr [[TMP51]], align 4, !tbaa [[TBAA28]] +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP62]], ptr [[TMP49]], align 4, !tbaa [[TBAA28]] ; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP48]]) #[[ATTR0]] ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP37]], ptr @PAYLOAD, align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP38]], ptr getelementptr inbounds ([37 x i32], ptr @PAYLOAD, i32 0, i32 22), align 4 @@ -121,56 +135,56 @@ define void @Miss(%struct.OuterPayload* noalias nocapture %outerPayload) #0 !typ ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP44]], ptr getelementptr inbounds ([37 x i32], ptr @PAYLOAD, i32 0, i32 28), align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP45]], ptr getelementptr inbounds ([37 x i32], ptr @PAYLOAD, i32 0, i32 29), align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP57:%.*]] = getelementptr inbounds [[STRUCT_OUTERPAYLOAD]], ptr [[TMP2]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP58:%.*]] = getelementptr i32, ptr [[TMP57]], i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP59:%.*]] = getelementptr i32, ptr [[TMP58]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP60:%.*]] = load i32, ptr [[TMP59]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP60]], ptr getelementptr inbounds ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP61:%.*]] = getelementptr i32, ptr [[TMP58]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP62:%.*]] = load i32, ptr [[TMP61]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP62]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP63:%.*]] = getelementptr i32, ptr [[TMP58]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP64:%.*]] = load i32, ptr [[TMP63]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP64]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP65:%.*]] = getelementptr i32, ptr [[TMP58]], i64 3 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP63:%.*]] = getelementptr inbounds [[STRUCT_OUTERPAYLOAD]], ptr [[TMP2]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP64:%.*]] = getelementptr i32, ptr [[TMP63]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP65:%.*]] = getelementptr i32, ptr [[TMP64]], i64 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP66:%.*]] = load i32, ptr [[TMP65]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP66]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 10), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP67:%.*]] = getelementptr i32, ptr [[TMP58]], i64 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP66]], ptr getelementptr inbounds ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP67:%.*]] = getelementptr i32, ptr [[TMP64]], i64 1 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP68:%.*]] = load i32, ptr [[TMP67]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP68]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 11), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP69:%.*]] = getelementptr i32, ptr [[TMP58]], i64 5 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP68]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP69:%.*]] = getelementptr i32, ptr [[TMP64]], i64 2 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP70:%.*]] = load i32, ptr [[TMP69]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP70]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 12), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP71:%.*]] = getelementptr i32, ptr [[TMP58]], i64 6 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP70]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP71:%.*]] = getelementptr i32, ptr [[TMP64]], i64 3 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP72:%.*]] = load i32, ptr [[TMP71]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP72]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 13), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP73:%.*]] = getelementptr i32, ptr [[TMP58]], i64 7 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP72]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 10), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP73:%.*]] = getelementptr i32, ptr [[TMP64]], i64 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP74:%.*]] = load i32, ptr [[TMP73]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP74]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 14), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP75:%.*]] = getelementptr i32, ptr [[TMP58]], i64 8 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP74]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 11), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP75:%.*]] = getelementptr i32, ptr [[TMP64]], i64 5 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP76:%.*]] = load i32, ptr [[TMP75]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP76]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 15), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP77:%.*]] = getelementptr i32, ptr [[TMP58]], i64 9 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP76]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 12), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP77:%.*]] = getelementptr i32, ptr [[TMP64]], i64 6 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP78:%.*]] = load i32, ptr [[TMP77]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP78]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 16), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP79:%.*]] = getelementptr i32, ptr [[TMP58]], i64 10 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP78]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 13), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP79:%.*]] = getelementptr i32, ptr [[TMP64]], i64 7 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP80:%.*]] = load i32, ptr [[TMP79]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP80]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 17), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP81:%.*]] = getelementptr i32, ptr [[TMP58]], i64 11 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP80]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 14), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP81:%.*]] = getelementptr i32, ptr [[TMP64]], i64 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP82:%.*]] = load i32, ptr [[TMP81]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP82]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 18), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP83:%.*]] = getelementptr i32, ptr [[TMP58]], i64 12 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP82]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 15), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP83:%.*]] = getelementptr i32, ptr [[TMP64]], i64 9 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP84:%.*]] = load i32, ptr [[TMP83]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP84]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 19), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP85:%.*]] = getelementptr i32, ptr [[TMP58]], i64 13 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP84]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 16), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP85:%.*]] = getelementptr i32, ptr [[TMP64]], i64 10 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP86:%.*]] = load i32, ptr [[TMP85]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP86]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 20), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP87:%.*]] = getelementptr i32, ptr [[TMP58]], i64 14 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP86]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 17), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP87:%.*]] = getelementptr i32, ptr [[TMP64]], i64 11 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP88:%.*]] = load i32, ptr [[TMP87]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP88]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 21), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP89:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP90:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP89]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP90]], !continuation.registercount !24 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP88]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 18), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP89:%.*]] = getelementptr i32, ptr [[TMP64]], i64 12 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP90:%.*]] = load i32, ptr [[TMP89]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP90]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 19), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP91:%.*]] = getelementptr i32, ptr [[TMP64]], i64 13 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP92:%.*]] = load i32, ptr [[TMP91]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP92]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 20), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP93:%.*]] = getelementptr i32, ptr [[TMP64]], i64 14 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP94:%.*]] = load i32, ptr [[TMP93]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP94]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 21), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP95:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP96:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[TMP95]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP96]], !continuation.registercount [[META24]] ; %1 = load %dx.types.Handle, %dx.types.Handle* @"\01?myAccelerationStructure@@3URaytracingAccelerationStructure@@A", align 4 %2 = alloca %struct.InnerPayload, align 4 @@ -192,14 +206,13 @@ define void @Miss(%struct.OuterPayload* noalias nocapture %outerPayload) #0 !typ ; Function Attrs: nounwind define void @Callable(%struct.OuterPayload* noalias %outerPayload) #0 !types !23 { ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.DispatchSystemData @Callable( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR0]] !lgc.rt.shaderstage !32 !continuation.registercount !24 !continuation !33 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR0]] !lgc.rt.shaderstage [[META33:![0-9]+]] !continuation.registercount [[META24]] !continuation [[META34:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_OUTERPAYLOAD:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_OUTERPAYLOAD]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP4]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_OUTERPAYLOAD]], ptr [[TMP3]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[TMP5]], i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i64 0 @@ -415,164 +428,359 @@ define void @Callable(%struct.OuterPayload* noalias %outerPayload) #0 !types !23 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP157:%.*]] = getelementptr inbounds [[STRUCT_OUTERPAYLOAD]], ptr [[TMP3]], i32 0, i32 1, i32 14 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP158:%.*]] = load float, ptr [[TMP157]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP158]], ptr [[TMP156]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_CallShader.struct.OuterPayload(ptr [[SYSTEM_DATA_ALLOCA]], i32 0, ptr [[TMP2]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP159:%.*]] = load float, ptr [[TMP69]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP159]], ptr [[TMP70]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP160:%.*]] = load float, ptr [[TMP72]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP160]], ptr [[TMP73]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP161:%.*]] = load float, ptr [[TMP75]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP161]], ptr [[TMP76]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP162:%.*]] = load float, ptr [[TMP78]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP162]], ptr [[TMP79]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP163:%.*]] = load float, ptr [[TMP81]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP163]], ptr [[TMP82]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP164:%.*]] = load float, ptr [[TMP84]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP164]], ptr [[TMP85]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP165:%.*]] = load float, ptr [[TMP87]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP165]], ptr [[TMP88]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP166:%.*]] = load float, ptr [[TMP90]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP166]], ptr [[TMP91]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP167:%.*]] = load float, ptr [[TMP93]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP167]], ptr [[TMP94]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP168:%.*]] = load float, ptr [[TMP96]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP168]], ptr [[TMP97]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP169:%.*]] = load float, ptr [[TMP99]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP169]], ptr [[TMP100]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP170:%.*]] = load float, ptr [[TMP102]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP170]], ptr [[TMP103]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP171:%.*]] = load float, ptr [[TMP105]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP171]], ptr [[TMP106]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP172:%.*]] = load float, ptr [[TMP108]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP172]], ptr [[TMP109]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP173:%.*]] = load float, ptr [[TMP111]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP173]], ptr [[TMP112]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP174:%.*]] = load float, ptr [[TMP114]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP174]], ptr [[TMP115]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP175:%.*]] = load float, ptr [[TMP117]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP175]], ptr [[TMP118]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP176:%.*]] = load float, ptr [[TMP120]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP176]], ptr [[TMP121]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP177:%.*]] = load float, ptr [[TMP123]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP177]], ptr [[TMP124]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP178:%.*]] = load float, ptr [[TMP126]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP178]], ptr [[TMP127]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP179:%.*]] = load float, ptr [[TMP129]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP179]], ptr [[TMP130]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP180:%.*]] = load float, ptr [[TMP132]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP180]], ptr [[TMP133]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP181:%.*]] = load float, ptr [[TMP135]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP181]], ptr [[TMP136]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP182:%.*]] = load float, ptr [[TMP138]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP182]], ptr [[TMP139]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP183:%.*]] = load float, ptr [[TMP141]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP183]], ptr [[TMP142]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP184:%.*]] = load float, ptr [[TMP144]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP184]], ptr [[TMP145]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP185:%.*]] = load float, ptr [[TMP147]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP185]], ptr [[TMP148]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP186:%.*]] = load float, ptr [[TMP150]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP186]], ptr [[TMP151]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP187:%.*]] = load float, ptr [[TMP153]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP187]], ptr [[TMP154]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP188:%.*]] = load float, ptr [[TMP156]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP188]], ptr [[TMP157]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP189:%.*]] = getelementptr inbounds [[STRUCT_OUTERPAYLOAD]], ptr [[TMP3]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP190:%.*]] = getelementptr i32, ptr [[TMP189]], i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP191:%.*]] = getelementptr i32, ptr [[TMP190]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP192:%.*]] = load i32, ptr [[TMP191]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP192]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP193:%.*]] = getelementptr i32, ptr [[TMP190]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA_I:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP159:%.*]] = getelementptr inbounds [[STRUCT_OUTERPAYLOAD]], ptr [[TMP2]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP160:%.*]] = getelementptr i32, ptr [[TMP159]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP161:%.*]] = getelementptr i32, ptr [[TMP160]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP162:%.*]] = load i32, ptr [[TMP161]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP162]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP163:%.*]] = getelementptr i32, ptr [[TMP160]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP164:%.*]] = load i32, ptr [[TMP163]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP164]], ptr getelementptr (i32, ptr @PAYLOAD, i64 1), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP165:%.*]] = getelementptr i32, ptr [[TMP160]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP166:%.*]] = load i32, ptr [[TMP165]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP166]], ptr getelementptr (i32, ptr @PAYLOAD, i64 2), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP167:%.*]] = getelementptr i32, ptr [[TMP160]], i64 3 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP168:%.*]] = load i32, ptr [[TMP167]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP168]], ptr getelementptr (i32, ptr @PAYLOAD, i64 3), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP169:%.*]] = getelementptr i32, ptr [[TMP160]], i64 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP170:%.*]] = load i32, ptr [[TMP169]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP170]], ptr getelementptr (i32, ptr @PAYLOAD, i64 4), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP171:%.*]] = getelementptr i32, ptr [[TMP160]], i64 5 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP172:%.*]] = load i32, ptr [[TMP171]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP172]], ptr getelementptr (i32, ptr @PAYLOAD, i64 5), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP173:%.*]] = getelementptr i32, ptr [[TMP160]], i64 6 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP174:%.*]] = load i32, ptr [[TMP173]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP174]], ptr getelementptr (i32, ptr @PAYLOAD, i64 6), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP175:%.*]] = getelementptr i32, ptr [[TMP160]], i64 7 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP176:%.*]] = load i32, ptr [[TMP175]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP176]], ptr getelementptr (i32, ptr @PAYLOAD, i64 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP177:%.*]] = getelementptr i32, ptr [[TMP160]], i64 8 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP178:%.*]] = load i32, ptr [[TMP177]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP178]], ptr getelementptr (i32, ptr @PAYLOAD, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP179:%.*]] = getelementptr i32, ptr [[TMP160]], i64 9 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP180:%.*]] = load i32, ptr [[TMP179]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP180]], ptr getelementptr (i32, ptr @PAYLOAD, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP181:%.*]] = getelementptr i32, ptr [[TMP160]], i64 10 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP182:%.*]] = load i32, ptr [[TMP181]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP182]], ptr getelementptr (i32, ptr @PAYLOAD, i64 10), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP183:%.*]] = getelementptr i32, ptr [[TMP160]], i64 11 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP184:%.*]] = load i32, ptr [[TMP183]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP184]], ptr getelementptr (i32, ptr @PAYLOAD, i64 11), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP185:%.*]] = getelementptr i32, ptr [[TMP160]], i64 12 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP186:%.*]] = load i32, ptr [[TMP185]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP186]], ptr getelementptr (i32, ptr @PAYLOAD, i64 12), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP187:%.*]] = getelementptr i32, ptr [[TMP160]], i64 13 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP188:%.*]] = load i32, ptr [[TMP187]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP188]], ptr getelementptr (i32, ptr @PAYLOAD, i64 13), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP189:%.*]] = getelementptr i32, ptr [[TMP160]], i64 14 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP190:%.*]] = load i32, ptr [[TMP189]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP190]], ptr getelementptr (i32, ptr @PAYLOAD, i64 14), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP191:%.*]] = getelementptr inbounds [[STRUCT_OUTERPAYLOAD]], ptr [[TMP2]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP192:%.*]] = getelementptr i32, ptr [[TMP191]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP193:%.*]] = getelementptr i32, ptr [[TMP192]], i64 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP194:%.*]] = load i32, ptr [[TMP193]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP194]], ptr getelementptr (i32, ptr @PAYLOAD, i64 1), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP195:%.*]] = getelementptr i32, ptr [[TMP190]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP194]], ptr getelementptr inbounds ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i32 15), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP195:%.*]] = getelementptr i32, ptr [[TMP192]], i64 1 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP196:%.*]] = load i32, ptr [[TMP195]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP196]], ptr getelementptr (i32, ptr @PAYLOAD, i64 2), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP197:%.*]] = getelementptr i32, ptr [[TMP190]], i64 3 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP196]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 16), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP197:%.*]] = getelementptr i32, ptr [[TMP192]], i64 2 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP198:%.*]] = load i32, ptr [[TMP197]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP198]], ptr getelementptr (i32, ptr @PAYLOAD, i64 3), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP199:%.*]] = getelementptr i32, ptr [[TMP190]], i64 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP198]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 17), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP199:%.*]] = getelementptr i32, ptr [[TMP192]], i64 3 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP200:%.*]] = load i32, ptr [[TMP199]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP200]], ptr getelementptr (i32, ptr @PAYLOAD, i64 4), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP201:%.*]] = getelementptr i32, ptr [[TMP190]], i64 5 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP200]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 18), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP201:%.*]] = getelementptr i32, ptr [[TMP192]], i64 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP202:%.*]] = load i32, ptr [[TMP201]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP202]], ptr getelementptr (i32, ptr @PAYLOAD, i64 5), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP203:%.*]] = getelementptr i32, ptr [[TMP190]], i64 6 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP202]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 19), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP203:%.*]] = getelementptr i32, ptr [[TMP192]], i64 5 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP204:%.*]] = load i32, ptr [[TMP203]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP204]], ptr getelementptr (i32, ptr @PAYLOAD, i64 6), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP205:%.*]] = getelementptr i32, ptr [[TMP190]], i64 7 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP204]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 20), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP205:%.*]] = getelementptr i32, ptr [[TMP192]], i64 6 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP206:%.*]] = load i32, ptr [[TMP205]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP206]], ptr getelementptr (i32, ptr @PAYLOAD, i64 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP207:%.*]] = getelementptr i32, ptr [[TMP190]], i64 8 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP206]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 21), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP207:%.*]] = getelementptr i32, ptr [[TMP192]], i64 7 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP208:%.*]] = load i32, ptr [[TMP207]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP208]], ptr getelementptr (i32, ptr @PAYLOAD, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP209:%.*]] = getelementptr i32, ptr [[TMP190]], i64 9 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP208]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 22), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP209:%.*]] = getelementptr i32, ptr [[TMP192]], i64 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP210:%.*]] = load i32, ptr [[TMP209]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP210]], ptr getelementptr (i32, ptr @PAYLOAD, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP211:%.*]] = getelementptr i32, ptr [[TMP190]], i64 10 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP210]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 23), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP211:%.*]] = getelementptr i32, ptr [[TMP192]], i64 9 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP212:%.*]] = load i32, ptr [[TMP211]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP212]], ptr getelementptr (i32, ptr @PAYLOAD, i64 10), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP213:%.*]] = getelementptr i32, ptr [[TMP190]], i64 11 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP212]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 24), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP213:%.*]] = getelementptr i32, ptr [[TMP192]], i64 10 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP214:%.*]] = load i32, ptr [[TMP213]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP214]], ptr getelementptr (i32, ptr @PAYLOAD, i64 11), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP215:%.*]] = getelementptr i32, ptr [[TMP190]], i64 12 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP214]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 25), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP215:%.*]] = getelementptr i32, ptr [[TMP192]], i64 11 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP216:%.*]] = load i32, ptr [[TMP215]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP216]], ptr getelementptr (i32, ptr @PAYLOAD, i64 12), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP217:%.*]] = getelementptr i32, ptr [[TMP190]], i64 13 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP216]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 26), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP217:%.*]] = getelementptr i32, ptr [[TMP192]], i64 12 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP218:%.*]] = load i32, ptr [[TMP217]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP218]], ptr getelementptr (i32, ptr @PAYLOAD, i64 13), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP219:%.*]] = getelementptr i32, ptr [[TMP190]], i64 14 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP218]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 27), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP219:%.*]] = getelementptr i32, ptr [[TMP192]], i64 13 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP220:%.*]] = load i32, ptr [[TMP219]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP220]], ptr getelementptr (i32, ptr @PAYLOAD, i64 14), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP221:%.*]] = getelementptr inbounds [[STRUCT_OUTERPAYLOAD]], ptr [[TMP3]], i32 0, i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP222:%.*]] = getelementptr i32, ptr [[TMP221]], i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP223:%.*]] = getelementptr i32, ptr [[TMP222]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP224:%.*]] = load i32, ptr [[TMP223]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP224]], ptr getelementptr inbounds ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i32 15), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP225:%.*]] = getelementptr i32, ptr [[TMP222]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP226:%.*]] = load i32, ptr [[TMP225]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP226]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 16), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP227:%.*]] = getelementptr i32, ptr [[TMP222]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP228:%.*]] = load i32, ptr [[TMP227]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP228]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 17), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP229:%.*]] = getelementptr i32, ptr [[TMP222]], i64 3 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP230:%.*]] = load i32, ptr [[TMP229]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP230]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 18), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP231:%.*]] = getelementptr i32, ptr [[TMP222]], i64 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP232:%.*]] = load i32, ptr [[TMP231]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP232]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 19), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP233:%.*]] = getelementptr i32, ptr [[TMP222]], i64 5 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP234:%.*]] = load i32, ptr [[TMP233]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP234]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 20), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP235:%.*]] = getelementptr i32, ptr [[TMP222]], i64 6 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP236:%.*]] = load i32, ptr [[TMP235]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP236]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 21), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP237:%.*]] = getelementptr i32, ptr [[TMP222]], i64 7 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP238:%.*]] = load i32, ptr [[TMP237]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP238]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 22), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP239:%.*]] = getelementptr i32, ptr [[TMP222]], i64 8 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP240:%.*]] = load i32, ptr [[TMP239]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP240]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 23), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP241:%.*]] = getelementptr i32, ptr [[TMP222]], i64 9 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP242:%.*]] = load i32, ptr [[TMP241]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP242]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 24), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP243:%.*]] = getelementptr i32, ptr [[TMP222]], i64 10 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP244:%.*]] = load i32, ptr [[TMP243]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP244]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 25), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP245:%.*]] = getelementptr i32, ptr [[TMP222]], i64 11 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP246:%.*]] = load i32, ptr [[TMP245]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP246]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 26), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP247:%.*]] = getelementptr i32, ptr [[TMP222]], i64 12 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP248:%.*]] = load i32, ptr [[TMP247]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP248]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 27), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP249:%.*]] = getelementptr i32, ptr [[TMP222]], i64 13 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP250:%.*]] = load i32, ptr [[TMP249]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP250]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 28), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP251:%.*]] = getelementptr i32, ptr [[TMP222]], i64 14 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP252:%.*]] = load i32, ptr [[TMP251]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP252]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 29), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP253:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP253]], !continuation.registercount !24 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP220]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 28), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP221:%.*]] = getelementptr i32, ptr [[TMP192]], i64 14 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP222:%.*]] = load i32, ptr [[TMP221]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP222]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 29), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP223:%.*]] = call ptr inttoptr (i64 2 to ptr)([[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I]]), !continuation.registercount [[META24]], !continuation.returnedRegistercount !24 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP224:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @await.struct.DispatchSystemData(ptr [[TMP223]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_OUTERPAYLOAD]] poison, ptr [[TMP2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP225:%.*]] = getelementptr inbounds [[STRUCT_OUTERPAYLOAD]], ptr [[TMP2]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP226:%.*]] = getelementptr i32, ptr [[TMP225]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP227:%.*]] = getelementptr i32, ptr [[TMP226]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP228:%.*]] = load i32, ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP228]], ptr [[TMP227]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP229:%.*]] = getelementptr i32, ptr [[TMP226]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP230:%.*]] = load i32, ptr getelementptr (i32, ptr @PAYLOAD, i64 1), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP230]], ptr [[TMP229]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP231:%.*]] = getelementptr i32, ptr [[TMP226]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP232:%.*]] = load i32, ptr getelementptr (i32, ptr @PAYLOAD, i64 2), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP232]], ptr [[TMP231]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP233:%.*]] = getelementptr i32, ptr [[TMP226]], i64 3 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP234:%.*]] = load i32, ptr getelementptr (i32, ptr @PAYLOAD, i64 3), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP234]], ptr [[TMP233]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP235:%.*]] = getelementptr i32, ptr [[TMP226]], i64 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP236:%.*]] = load i32, ptr getelementptr (i32, ptr @PAYLOAD, i64 4), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP236]], ptr [[TMP235]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP237:%.*]] = getelementptr i32, ptr [[TMP226]], i64 5 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP238:%.*]] = load i32, ptr getelementptr (i32, ptr @PAYLOAD, i64 5), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP238]], ptr [[TMP237]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP239:%.*]] = getelementptr i32, ptr [[TMP226]], i64 6 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP240:%.*]] = load i32, ptr getelementptr (i32, ptr @PAYLOAD, i64 6), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP240]], ptr [[TMP239]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP241:%.*]] = getelementptr i32, ptr [[TMP226]], i64 7 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP242:%.*]] = load i32, ptr getelementptr (i32, ptr @PAYLOAD, i64 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP242]], ptr [[TMP241]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP243:%.*]] = getelementptr i32, ptr [[TMP226]], i64 8 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP244:%.*]] = load i32, ptr getelementptr (i32, ptr @PAYLOAD, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP244]], ptr [[TMP243]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP245:%.*]] = getelementptr i32, ptr [[TMP226]], i64 9 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP246:%.*]] = load i32, ptr getelementptr (i32, ptr @PAYLOAD, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP246]], ptr [[TMP245]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP247:%.*]] = getelementptr i32, ptr [[TMP226]], i64 10 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP248:%.*]] = load i32, ptr getelementptr (i32, ptr @PAYLOAD, i64 10), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP248]], ptr [[TMP247]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP249:%.*]] = getelementptr i32, ptr [[TMP226]], i64 11 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP250:%.*]] = load i32, ptr getelementptr (i32, ptr @PAYLOAD, i64 11), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP250]], ptr [[TMP249]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP251:%.*]] = getelementptr i32, ptr [[TMP226]], i64 12 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP252:%.*]] = load i32, ptr getelementptr (i32, ptr @PAYLOAD, i64 12), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP252]], ptr [[TMP251]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP253:%.*]] = getelementptr i32, ptr [[TMP226]], i64 13 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP254:%.*]] = load i32, ptr getelementptr (i32, ptr @PAYLOAD, i64 13), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP254]], ptr [[TMP253]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP255:%.*]] = getelementptr i32, ptr [[TMP226]], i64 14 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP256:%.*]] = load i32, ptr getelementptr (i32, ptr @PAYLOAD, i64 14), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP256]], ptr [[TMP255]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP257:%.*]] = getelementptr inbounds [[STRUCT_OUTERPAYLOAD]], ptr [[TMP2]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP258:%.*]] = getelementptr i32, ptr [[TMP257]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP259:%.*]] = getelementptr i32, ptr [[TMP258]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP260:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i32 15), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP260]], ptr [[TMP259]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP261:%.*]] = getelementptr i32, ptr [[TMP258]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP262:%.*]] = load i32, ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 16), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP262]], ptr [[TMP261]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP263:%.*]] = getelementptr i32, ptr [[TMP258]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP264:%.*]] = load i32, ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 17), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP264]], ptr [[TMP263]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP265:%.*]] = getelementptr i32, ptr [[TMP258]], i64 3 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP266:%.*]] = load i32, ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 18), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP266]], ptr [[TMP265]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP267:%.*]] = getelementptr i32, ptr [[TMP258]], i64 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP268:%.*]] = load i32, ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 19), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP268]], ptr [[TMP267]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP269:%.*]] = getelementptr i32, ptr [[TMP258]], i64 5 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP270:%.*]] = load i32, ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 20), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP270]], ptr [[TMP269]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP271:%.*]] = getelementptr i32, ptr [[TMP258]], i64 6 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP272:%.*]] = load i32, ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 21), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP272]], ptr [[TMP271]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP273:%.*]] = getelementptr i32, ptr [[TMP258]], i64 7 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP274:%.*]] = load i32, ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 22), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP274]], ptr [[TMP273]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP275:%.*]] = getelementptr i32, ptr [[TMP258]], i64 8 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP276:%.*]] = load i32, ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 23), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP276]], ptr [[TMP275]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP277:%.*]] = getelementptr i32, ptr [[TMP258]], i64 9 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP278:%.*]] = load i32, ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 24), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP278]], ptr [[TMP277]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP279:%.*]] = getelementptr i32, ptr [[TMP258]], i64 10 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP280:%.*]] = load i32, ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 25), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP280]], ptr [[TMP279]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP281:%.*]] = getelementptr i32, ptr [[TMP258]], i64 11 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP282:%.*]] = load i32, ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 26), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP282]], ptr [[TMP281]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP283:%.*]] = getelementptr i32, ptr [[TMP258]], i64 12 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP284:%.*]] = load i32, ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 27), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP284]], ptr [[TMP283]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP285:%.*]] = getelementptr i32, ptr [[TMP258]], i64 13 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP286:%.*]] = load i32, ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 28), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP286]], ptr [[TMP285]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP287:%.*]] = getelementptr i32, ptr [[TMP258]], i64 14 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP288:%.*]] = load i32, ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 29), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP288]], ptr [[TMP287]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP224]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; LOWERRAYTRACINGPIPELINE-NEXT: br label [[DOTSPLIT:%.*]] +; LOWERRAYTRACINGPIPELINE: .split: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP289:%.*]] = load float, ptr [[TMP69]], align 8 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP289]], ptr [[TMP70]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP290:%.*]] = load float, ptr [[TMP72]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP290]], ptr [[TMP73]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP291:%.*]] = load float, ptr [[TMP75]], align 8 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP291]], ptr [[TMP76]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP292:%.*]] = load float, ptr [[TMP78]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP292]], ptr [[TMP79]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP293:%.*]] = load float, ptr [[TMP81]], align 8 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP293]], ptr [[TMP82]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP294:%.*]] = load float, ptr [[TMP84]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP294]], ptr [[TMP85]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP295:%.*]] = load float, ptr [[TMP87]], align 8 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP295]], ptr [[TMP88]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP296:%.*]] = load float, ptr [[TMP90]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP296]], ptr [[TMP91]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP297:%.*]] = load float, ptr [[TMP93]], align 8 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP297]], ptr [[TMP94]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP298:%.*]] = load float, ptr [[TMP96]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP298]], ptr [[TMP97]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP299:%.*]] = load float, ptr [[TMP99]], align 8 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP299]], ptr [[TMP100]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP300:%.*]] = load float, ptr [[TMP102]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP300]], ptr [[TMP103]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP301:%.*]] = load float, ptr [[TMP105]], align 8 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP301]], ptr [[TMP106]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP302:%.*]] = load float, ptr [[TMP108]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP302]], ptr [[TMP109]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP303:%.*]] = load float, ptr [[TMP111]], align 8 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP303]], ptr [[TMP112]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP304:%.*]] = load float, ptr [[TMP114]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP304]], ptr [[TMP115]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP305:%.*]] = load float, ptr [[TMP117]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP305]], ptr [[TMP118]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP306:%.*]] = load float, ptr [[TMP120]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP306]], ptr [[TMP121]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP307:%.*]] = load float, ptr [[TMP123]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP307]], ptr [[TMP124]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP308:%.*]] = load float, ptr [[TMP126]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP308]], ptr [[TMP127]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP309:%.*]] = load float, ptr [[TMP129]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP309]], ptr [[TMP130]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP310:%.*]] = load float, ptr [[TMP132]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP310]], ptr [[TMP133]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP311:%.*]] = load float, ptr [[TMP135]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP311]], ptr [[TMP136]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP312:%.*]] = load float, ptr [[TMP138]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP312]], ptr [[TMP139]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP313:%.*]] = load float, ptr [[TMP141]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP313]], ptr [[TMP142]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP314:%.*]] = load float, ptr [[TMP144]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP314]], ptr [[TMP145]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP315:%.*]] = load float, ptr [[TMP147]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP315]], ptr [[TMP148]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP316:%.*]] = load float, ptr [[TMP150]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP316]], ptr [[TMP151]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP317:%.*]] = load float, ptr [[TMP153]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP317]], ptr [[TMP154]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP318:%.*]] = load float, ptr [[TMP156]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store float [[TMP318]], ptr [[TMP157]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP319:%.*]] = getelementptr inbounds [[STRUCT_OUTERPAYLOAD]], ptr [[TMP3]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP320:%.*]] = getelementptr i32, ptr [[TMP319]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP321:%.*]] = getelementptr i32, ptr [[TMP320]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP322:%.*]] = load i32, ptr [[TMP321]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP322]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP323:%.*]] = getelementptr i32, ptr [[TMP320]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP324:%.*]] = load i32, ptr [[TMP323]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP324]], ptr getelementptr (i32, ptr @PAYLOAD, i64 1), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP325:%.*]] = getelementptr i32, ptr [[TMP320]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP326:%.*]] = load i32, ptr [[TMP325]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP326]], ptr getelementptr (i32, ptr @PAYLOAD, i64 2), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP327:%.*]] = getelementptr i32, ptr [[TMP320]], i64 3 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP328:%.*]] = load i32, ptr [[TMP327]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP328]], ptr getelementptr (i32, ptr @PAYLOAD, i64 3), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP329:%.*]] = getelementptr i32, ptr [[TMP320]], i64 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP330:%.*]] = load i32, ptr [[TMP329]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP330]], ptr getelementptr (i32, ptr @PAYLOAD, i64 4), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP331:%.*]] = getelementptr i32, ptr [[TMP320]], i64 5 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP332:%.*]] = load i32, ptr [[TMP331]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP332]], ptr getelementptr (i32, ptr @PAYLOAD, i64 5), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP333:%.*]] = getelementptr i32, ptr [[TMP320]], i64 6 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP334:%.*]] = load i32, ptr [[TMP333]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP334]], ptr getelementptr (i32, ptr @PAYLOAD, i64 6), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP335:%.*]] = getelementptr i32, ptr [[TMP320]], i64 7 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP336:%.*]] = load i32, ptr [[TMP335]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP336]], ptr getelementptr (i32, ptr @PAYLOAD, i64 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP337:%.*]] = getelementptr i32, ptr [[TMP320]], i64 8 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP338:%.*]] = load i32, ptr [[TMP337]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP338]], ptr getelementptr (i32, ptr @PAYLOAD, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP339:%.*]] = getelementptr i32, ptr [[TMP320]], i64 9 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP340:%.*]] = load i32, ptr [[TMP339]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP340]], ptr getelementptr (i32, ptr @PAYLOAD, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP341:%.*]] = getelementptr i32, ptr [[TMP320]], i64 10 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP342:%.*]] = load i32, ptr [[TMP341]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP342]], ptr getelementptr (i32, ptr @PAYLOAD, i64 10), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP343:%.*]] = getelementptr i32, ptr [[TMP320]], i64 11 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP344:%.*]] = load i32, ptr [[TMP343]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP344]], ptr getelementptr (i32, ptr @PAYLOAD, i64 11), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP345:%.*]] = getelementptr i32, ptr [[TMP320]], i64 12 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP346:%.*]] = load i32, ptr [[TMP345]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP346]], ptr getelementptr (i32, ptr @PAYLOAD, i64 12), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP347:%.*]] = getelementptr i32, ptr [[TMP320]], i64 13 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP348:%.*]] = load i32, ptr [[TMP347]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP348]], ptr getelementptr (i32, ptr @PAYLOAD, i64 13), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP349:%.*]] = getelementptr i32, ptr [[TMP320]], i64 14 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP350:%.*]] = load i32, ptr [[TMP349]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP350]], ptr getelementptr (i32, ptr @PAYLOAD, i64 14), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP351:%.*]] = getelementptr inbounds [[STRUCT_OUTERPAYLOAD]], ptr [[TMP3]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP352:%.*]] = getelementptr i32, ptr [[TMP351]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP353:%.*]] = getelementptr i32, ptr [[TMP352]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP354:%.*]] = load i32, ptr [[TMP353]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP354]], ptr getelementptr inbounds ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i32 15), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP355:%.*]] = getelementptr i32, ptr [[TMP352]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP356:%.*]] = load i32, ptr [[TMP355]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP356]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 16), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP357:%.*]] = getelementptr i32, ptr [[TMP352]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP358:%.*]] = load i32, ptr [[TMP357]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP358]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 17), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP359:%.*]] = getelementptr i32, ptr [[TMP352]], i64 3 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP360:%.*]] = load i32, ptr [[TMP359]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP360]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 18), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP361:%.*]] = getelementptr i32, ptr [[TMP352]], i64 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP362:%.*]] = load i32, ptr [[TMP361]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP362]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 19), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP363:%.*]] = getelementptr i32, ptr [[TMP352]], i64 5 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP364:%.*]] = load i32, ptr [[TMP363]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP364]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 20), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP365:%.*]] = getelementptr i32, ptr [[TMP352]], i64 6 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP366:%.*]] = load i32, ptr [[TMP365]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP366]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 21), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP367:%.*]] = getelementptr i32, ptr [[TMP352]], i64 7 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP368:%.*]] = load i32, ptr [[TMP367]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP368]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 22), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP369:%.*]] = getelementptr i32, ptr [[TMP352]], i64 8 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP370:%.*]] = load i32, ptr [[TMP369]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP370]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 23), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP371:%.*]] = getelementptr i32, ptr [[TMP352]], i64 9 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP372:%.*]] = load i32, ptr [[TMP371]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP372]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 24), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP373:%.*]] = getelementptr i32, ptr [[TMP352]], i64 10 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP374:%.*]] = load i32, ptr [[TMP373]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP374]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 25), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP375:%.*]] = getelementptr i32, ptr [[TMP352]], i64 11 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP376:%.*]] = load i32, ptr [[TMP375]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP376]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 26), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP377:%.*]] = getelementptr i32, ptr [[TMP352]], i64 12 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP378:%.*]] = load i32, ptr [[TMP377]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP378]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 27), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP379:%.*]] = getelementptr i32, ptr [[TMP352]], i64 13 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP380:%.*]] = load i32, ptr [[TMP379]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP380]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 28), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP381:%.*]] = getelementptr i32, ptr [[TMP352]], i64 14 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP382:%.*]] = load i32, ptr [[TMP381]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP382]], ptr getelementptr ([[STRUCT_OUTERPAYLOAD_LAYOUT_CALLSHADER]], ptr @PAYLOAD, i32 0, i32 0, i64 29), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP383:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP383]], !continuation.registercount [[META24]] ; %1 = alloca %struct.OuterPayload, align 8 %2 = getelementptr inbounds %struct.OuterPayload, %struct.OuterPayload* %1, i32 0, i32 0, i32 0 @@ -812,18 +1020,6 @@ define i32 @_cont_GetLocalRootIndex(%struct.DispatchSystemData* %data) #3 !types ; Function Attrs: alwaysinline define void @_cont_TraceRay(%struct.DispatchSystemData* %data, i64 %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, float %6, float %7, float %8, float %9, float %10, float %11, float %12, float %13) #3 !types !44 { -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_TraceRay( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i64 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]], i32 [[TMP3:%.*]], i32 [[TMP4:%.*]], i32 [[TMP5:%.*]], float [[TMP6:%.*]], float [[TMP7:%.*]], float [[TMP8:%.*]], float [[TMP9:%.*]], float [[TMP10:%.*]], float [[TMP11:%.*]], float [[TMP12:%.*]], float [[TMP13:%.*]]) #[[ATTR3]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[NEWDATA:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @_AmdAwaitTraversal(i64 4, [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; %dis_data = load %struct.DispatchSystemData, %struct.DispatchSystemData* %data, align 4 %sys_data = insertvalue %struct.SystemData undef, %struct.DispatchSystemData %dis_data, 0 %trav_data = insertvalue %struct.TraversalData undef, %struct.SystemData %sys_data, 0 @@ -835,16 +1031,6 @@ define void @_cont_TraceRay(%struct.DispatchSystemData* %data, i64 %0, i32 %1, i ; Function Attrs: alwaysinline define void @_cont_CallShader(%struct.DispatchSystemData* %data, i32 %0) #3 !types !45 { -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_CallShader( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i32 [[TMP0:%.*]]) #[[ATTR3]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[NEWDATA:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @_AmdAwaitShader(i64 2, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; %dis_data = load %struct.DispatchSystemData, %struct.DispatchSystemData* %data, align 4 %newdata = call %struct.DispatchSystemData @_AmdAwaitShader(i64 2, %struct.DispatchSystemData %dis_data) store %struct.DispatchSystemData %newdata, %struct.DispatchSystemData* %data, align 4 diff --git a/shared/continuations/test/dx/payload.ll b/shared/continuations/test/dx/payload.ll index 1fbdd7bc3a..5f460e66bf 100644 --- a/shared/continuations/test/dx/payload.ll +++ b/shared/continuations/test/dx/payload.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 3 -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,remove-types-metadata' -S %s 2>%t0.stderr | FileCheck -check-prefix=CLEANUP %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,remove-types-metadata' -S %s 2> %t0.stderr | FileCheck -check-prefix=CLEANUP %s ; RUN: count 0 < %t0.stderr -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' \ -; RUN: -S %s 2>%t1.stderr | FileCheck -check-prefix=POST-PROCESS %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' \ +; RUN: -S %s 2> %t1.stderr | FileCheck -check-prefix=POST-PROCESS %s ; RUN: count 0 < %t1.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" @@ -178,25 +178,28 @@ attributes #3 = { nounwind } ; CLEANUP-NEXT: ret i32 5 ; ; -; CLEANUP-LABEL: define void @main() !lgc.rt.shaderstage !10 !continuation.entry !21 !continuation.registercount !10 !continuation !22 !continuation.stacksize !23 !continuation.state !10 { +; CLEANUP-LABEL: define void @main() !lgc.rt.shaderstage !9 !continuation.entry !20 !continuation.registercount !9 !continuation !21 !continuation.stacksize !22 !continuation.state !9 { ; CLEANUP-NEXT: AllocaSpillBB: ; CLEANUP-NEXT: [[CONT_STATE:%.*]] = alloca [0 x i32], align 4 ; CLEANUP-NEXT: [[TMP0:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA:%.*]] @continuations.getSystemData.s_struct.DispatchSystemDatas() ; CLEANUP-NEXT: [[DOTFCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP0]], 0 ; CLEANUP-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) -; CLEANUP-NEXT: [[TMP1:%.*]] = load [[DX_TYPES_HANDLE:%.*]], ptr @"\01?Scene@@3URaytracingAccelerationStructure@@A", align 4 -; CLEANUP-NEXT: [[TMP2:%.*]] = load [[DX_TYPES_HANDLE]], ptr @"\01?RenderTarget@@3V?$RWTexture2D@V?$vector@M$03@@@@A", align 4 -; CLEANUP-NEXT: [[TMP3:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP1]]) -; CLEANUP-NEXT: [[TMP4:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP3]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) -; CLEANUP-NEXT: [[TMP5:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[TMP4]]) +; CLEANUP-NEXT: [[TMP1:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4 +; CLEANUP-NEXT: [[TMP3:%.*]] = add i32 [[TMP2]], 108 +; CLEANUP-NEXT: store i32 [[TMP3]], ptr [[TMP1]], align 4 +; CLEANUP-NEXT: [[TMP4:%.*]] = load [[DX_TYPES_HANDLE:%.*]], ptr @"\01?Scene@@3URaytracingAccelerationStructure@@A", align 4 +; CLEANUP-NEXT: [[TMP5:%.*]] = load [[DX_TYPES_HANDLE]], ptr @"\01?RenderTarget@@3V?$RWTexture2D@V?$vector@M$03@@@@A", align 4 +; CLEANUP-NEXT: [[TMP6:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP4]]) +; CLEANUP-NEXT: [[TMP7:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP6]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) +; CLEANUP-NEXT: [[TMP8:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[TMP7]]) ; CLEANUP-NEXT: [[DIS_DATA_I_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, i32 [[DOTFCA_0_EXTRACT]], 0 ; CLEANUP-NEXT: [[SYS_DATA_I:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I_FCA_0_INSERT]], 0 ; CLEANUP-NEXT: [[TRAV_DATA_I:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA_I]], 0 -; CLEANUP-NEXT: [[TMP6:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4 -; CLEANUP-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], 108 -; CLEANUP-NEXT: store i32 [[TMP8]], ptr [[TMP6]], align 4 -; CLEANUP-NEXT: store i32 [[TMP7]], ptr @PAYLOAD, align 4 +; CLEANUP-NEXT: [[TMP9:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4 +; CLEANUP-NEXT: [[TMP11:%.*]] = add i32 [[TMP10]], -108 +; CLEANUP-NEXT: store i32 [[TMP11]], ptr @PAYLOAD, align 4 ; CLEANUP-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) ; CLEANUP-NEXT: store i32 undef, ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 ; CLEANUP-NEXT: store i32 undef, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 @@ -248,14 +251,14 @@ attributes #3 = { nounwind } ; CLEANUP-NEXT: store i32 undef, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 54), align 4 ; CLEANUP-NEXT: store i32 undef, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 55), align 4 ; CLEANUP-NEXT: store i32 undef, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 56), align 4 -; CLEANUP-NEXT: [[TMP9:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4 -; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 4, i32 [[TMP10]], i64 ptrtoint (ptr @main.resume.0 to i64), [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]]), !continuation.registercount !19, !continuation.returnedRegistercount !19 +; CLEANUP-NEXT: [[TMP12:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4 +; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 4, i32 [[TMP13]], i64 ptrtoint (ptr @main.resume.0 to i64), [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]]), !continuation.registercount !18, !continuation.returnedRegistercount !18 ; CLEANUP-NEXT: unreachable ; ; -; CLEANUP-LABEL: define void @main.resume.0( -; CLEANUP-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage !10 !continuation.registercount !19 !continuation !22 { +; CLEANUP-LABEL: define dso_local void @main.resume.0( +; CLEANUP-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage !9 !continuation.registercount !18 !continuation !21 { ; CLEANUP-NEXT: entryresume.0: ; CLEANUP-NEXT: [[CONT_STATE:%.*]] = alloca [0 x i32], align 4 ; CLEANUP-NEXT: [[TMP2:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 @@ -308,19 +311,18 @@ attributes #3 = { nounwind } ; CLEANUP-NEXT: [[TMP49:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 54), align 4 ; CLEANUP-NEXT: [[TMP50:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 55), align 4 ; CLEANUP-NEXT: [[TMP51:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 56), align 4 -; CLEANUP-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) +; CLEANUP-NEXT: [[DOTFCA_0_EXTRACT1:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], 0 +; CLEANUP-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; CLEANUP-NEXT: [[TMP52:%.*]] = call ptr @continuation.getContinuationStackOffset() ; CLEANUP-NEXT: [[TMP53:%.*]] = load i32, ptr [[TMP52]], align 4 ; CLEANUP-NEXT: [[TMP54:%.*]] = add i32 [[TMP53]], -108 ; CLEANUP-NEXT: store i32 [[TMP54]], ptr [[TMP52]], align 4 -; CLEANUP-NEXT: [[DOTFCA_0_EXTRACT1:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], 0 -; CLEANUP-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; CLEANUP-NEXT: call void @continuation.complete() ; CLEANUP-NEXT: unreachable ; ; ; CLEANUP-LABEL: define void @AnyHit( -; CLEANUP-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] [[TMP1:%.*]]) #[[ATTR3:[0-9]+]] !lgc.rt.shaderstage !24 !continuation.registercount !19 !continuation !25 !continuation.state !10 { +; CLEANUP-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_ANYHITTRAVERSALDATA:%.*]] [[TMP0:%.*]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] [[TMP1:%.*]]) #[[ATTR3:[0-9]+]] !lgc.rt.shaderstage !23 !continuation.registercount !18 !continuation !24 !continuation.state !9 { ; CLEANUP-NEXT: AllocaSpillBB: ; CLEANUP-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_ANYHITTRAVERSALDATA]], align 8 ; CLEANUP-NEXT: [[CONT_STATE:%.*]] = alloca [0 x i32], align 4 @@ -469,15 +471,14 @@ attributes #3 = { nounwind } ; CLEANUP-NEXT: [[DOTFCA_1_1_INSERT:%.*]] = insertvalue [[STRUCT_ANYHITTRAVERSALDATA]] [[DOTFCA_1_0_INSERT]], i32 [[DOTFCA_1_1_LOAD]], 1, 1 ; CLEANUP-NEXT: [[TMP63:%.*]] = call ptr @continuation.getContinuationStackOffset() ; CLEANUP-NEXT: [[TMP64:%.*]] = load i32, ptr [[TMP63]], align 4 -; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP64]], [[STRUCT_ANYHITTRAVERSALDATA]] [[DOTFCA_1_1_INSERT]]), !continuation.registercount !19 +; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP64]], [[STRUCT_ANYHITTRAVERSALDATA]] [[DOTFCA_1_1_INSERT]]), !continuation.registercount !18 ; CLEANUP-NEXT: unreachable ; ; ; CLEANUP-LABEL: define void @ClosestHit( -; CLEANUP-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR3]] !lgc.rt.shaderstage !26 !continuation.registercount !19 !continuation !27 !continuation.stacksize !28 !continuation.state !29 { +; CLEANUP-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR3]] !lgc.rt.shaderstage !25 !continuation.registercount !18 !continuation !26 !continuation.stacksize !27 !continuation.state !28 { ; CLEANUP-NEXT: AllocaSpillBB: ; CLEANUP-NEXT: [[CONT_STATE:%.*]] = alloca [4 x i32], align 4 -; CLEANUP-NEXT: call void @continuation.save.continuation_state() ; CLEANUP-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = getelementptr inbounds [[CLOSESTHIT_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; CLEANUP-NEXT: [[RETURNADDR_SPILL_ADDR:%.*]] = getelementptr inbounds [[CLOSESTHIT_FRAME]], ptr [[CONT_STATE]], i32 0, i32 1 ; CLEANUP-NEXT: store i64 [[RETURNADDR]], ptr [[RETURNADDR_SPILL_ADDR]], align 4 @@ -487,238 +488,245 @@ attributes #3 = { nounwind } ; CLEANUP-NEXT: store i32 [[DOTFCA_0_0_EXTRACT]], ptr [[DOTFCA_0_0_GEP]], align 4 ; CLEANUP-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 ; CLEANUP-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) -; CLEANUP-NEXT: [[TMP3:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; CLEANUP-NEXT: [[TMP4:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; CLEANUP-NEXT: [[TMP5:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; CLEANUP-NEXT: [[TMP6:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 10), align 4 -; CLEANUP-NEXT: [[TMP7:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 11), align 4 -; CLEANUP-NEXT: [[TMP8:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 12), align 4 -; CLEANUP-NEXT: [[TMP9:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 13), align 4 -; CLEANUP-NEXT: [[TMP10:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 14), align 4 -; CLEANUP-NEXT: [[TMP11:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 15), align 4 -; CLEANUP-NEXT: [[TMP12:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 16), align 4 -; CLEANUP-NEXT: [[TMP13:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 17), align 4 -; CLEANUP-NEXT: [[TMP14:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 18), align 4 -; CLEANUP-NEXT: [[TMP15:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 19), align 4 -; CLEANUP-NEXT: [[TMP16:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 20), align 4 -; CLEANUP-NEXT: [[TMP17:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 21), align 4 -; CLEANUP-NEXT: [[TMP18:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 22), align 4 -; CLEANUP-NEXT: [[TMP19:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 23), align 4 -; CLEANUP-NEXT: [[TMP20:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 24), align 4 -; CLEANUP-NEXT: [[TMP21:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 25), align 4 -; CLEANUP-NEXT: [[TMP22:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 26), align 4 -; CLEANUP-NEXT: [[TMP23:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 27), align 4 -; CLEANUP-NEXT: [[TMP24:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 28), align 4 -; CLEANUP-NEXT: [[TMP25:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 29), align 4 -; CLEANUP-NEXT: [[TMP26:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 30), align 4 -; CLEANUP-NEXT: [[TMP27:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 31), align 4 -; CLEANUP-NEXT: [[TMP28:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 32), align 4 -; CLEANUP-NEXT: [[TMP29:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 33), align 4 -; CLEANUP-NEXT: [[TMP30:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 34), align 4 -; CLEANUP-NEXT: [[TMP31:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 35), align 4 -; CLEANUP-NEXT: [[TMP32:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 36), align 4 -; CLEANUP-NEXT: [[TMP33:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 37), align 4 -; CLEANUP-NEXT: [[TMP34:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 38), align 4 -; CLEANUP-NEXT: [[TMP35:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 39), align 4 -; CLEANUP-NEXT: [[TMP36:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 40), align 4 -; CLEANUP-NEXT: [[TMP37:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 41), align 4 -; CLEANUP-NEXT: [[TMP38:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 42), align 4 -; CLEANUP-NEXT: [[TMP39:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 43), align 4 -; CLEANUP-NEXT: [[TMP40:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 44), align 4 -; CLEANUP-NEXT: [[TMP41:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 45), align 4 -; CLEANUP-NEXT: [[TMP42:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 46), align 4 -; CLEANUP-NEXT: [[TMP43:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 47), align 4 -; CLEANUP-NEXT: [[TMP44:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 48), align 4 -; CLEANUP-NEXT: [[TMP45:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 49), align 4 -; CLEANUP-NEXT: [[TMP46:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 50), align 4 -; CLEANUP-NEXT: [[TMP47:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 51), align 4 -; CLEANUP-NEXT: [[TMP48:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 52), align 4 -; CLEANUP-NEXT: [[TMP49:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 53), align 4 -; CLEANUP-NEXT: [[TMP50:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 54), align 4 -; CLEANUP-NEXT: [[TMP51:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 55), align 4 -; CLEANUP-NEXT: [[TMP52:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 56), align 4 +; CLEANUP-NEXT: [[TMP3:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4 +; CLEANUP-NEXT: [[TMP5:%.*]] = add i32 [[TMP4]], 108 +; CLEANUP-NEXT: store i32 [[TMP5]], ptr [[TMP3]], align 4 +; CLEANUP-NEXT: [[TMP6:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; CLEANUP-NEXT: [[TMP7:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; CLEANUP-NEXT: [[TMP8:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; CLEANUP-NEXT: [[TMP9:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 10), align 4 +; CLEANUP-NEXT: [[TMP10:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 11), align 4 +; CLEANUP-NEXT: [[TMP11:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 12), align 4 +; CLEANUP-NEXT: [[TMP12:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 13), align 4 +; CLEANUP-NEXT: [[TMP13:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 14), align 4 +; CLEANUP-NEXT: [[TMP14:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 15), align 4 +; CLEANUP-NEXT: [[TMP15:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 16), align 4 +; CLEANUP-NEXT: [[TMP16:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 17), align 4 +; CLEANUP-NEXT: [[TMP17:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 18), align 4 +; CLEANUP-NEXT: [[TMP18:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 19), align 4 +; CLEANUP-NEXT: [[TMP19:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 20), align 4 +; CLEANUP-NEXT: [[TMP20:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 21), align 4 +; CLEANUP-NEXT: [[TMP21:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 22), align 4 +; CLEANUP-NEXT: [[TMP22:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 23), align 4 +; CLEANUP-NEXT: [[TMP23:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 24), align 4 +; CLEANUP-NEXT: [[TMP24:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 25), align 4 +; CLEANUP-NEXT: [[TMP25:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 26), align 4 +; CLEANUP-NEXT: [[TMP26:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 27), align 4 +; CLEANUP-NEXT: [[TMP27:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 28), align 4 +; CLEANUP-NEXT: [[TMP28:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 29), align 4 +; CLEANUP-NEXT: [[TMP29:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 30), align 4 +; CLEANUP-NEXT: [[TMP30:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 31), align 4 +; CLEANUP-NEXT: [[TMP31:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 32), align 4 +; CLEANUP-NEXT: [[TMP32:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 33), align 4 +; CLEANUP-NEXT: [[TMP33:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 34), align 4 +; CLEANUP-NEXT: [[TMP34:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 35), align 4 +; CLEANUP-NEXT: [[TMP35:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 36), align 4 +; CLEANUP-NEXT: [[TMP36:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 37), align 4 +; CLEANUP-NEXT: [[TMP37:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 38), align 4 +; CLEANUP-NEXT: [[TMP38:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 39), align 4 +; CLEANUP-NEXT: [[TMP39:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 40), align 4 +; CLEANUP-NEXT: [[TMP40:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 41), align 4 +; CLEANUP-NEXT: [[TMP41:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 42), align 4 +; CLEANUP-NEXT: [[TMP42:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 43), align 4 +; CLEANUP-NEXT: [[TMP43:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 44), align 4 +; CLEANUP-NEXT: [[TMP44:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 45), align 4 +; CLEANUP-NEXT: [[TMP45:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 46), align 4 +; CLEANUP-NEXT: [[TMP46:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 47), align 4 +; CLEANUP-NEXT: [[TMP47:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 48), align 4 +; CLEANUP-NEXT: [[TMP48:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 49), align 4 +; CLEANUP-NEXT: [[TMP49:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 50), align 4 +; CLEANUP-NEXT: [[TMP50:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 51), align 4 +; CLEANUP-NEXT: [[TMP51:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 52), align 4 +; CLEANUP-NEXT: [[TMP52:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 53), align 4 +; CLEANUP-NEXT: [[TMP53:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 54), align 4 +; CLEANUP-NEXT: [[TMP54:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 55), align 4 +; CLEANUP-NEXT: [[TMP55:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 56), align 4 ; CLEANUP-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; CLEANUP-NEXT: [[TMP53:%.*]] = load i32, ptr @PAYLOAD, align 4 +; CLEANUP-NEXT: [[TMP56:%.*]] = load i32, ptr @PAYLOAD, align 4 ; CLEANUP-NEXT: [[DOTSPILL_ADDR:%.*]] = getelementptr inbounds [[CLOSESTHIT_FRAME]], ptr [[CONT_STATE]], i32 0, i32 2 -; CLEANUP-NEXT: store i32 [[TMP53]], ptr [[DOTSPILL_ADDR]], align 4 -; CLEANUP-NEXT: [[TMP54:%.*]] = call [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] @_cont_GetTriangleHitAttributes(ptr [[SYSTEM_DATA_ALLOCA]]) -; CLEANUP-NEXT: [[DOTFCA_0_EXTRACT52:%.*]] = extractvalue [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP54]], 0 +; CLEANUP-NEXT: store i32 [[TMP56]], ptr [[DOTSPILL_ADDR]], align 4 +; CLEANUP-NEXT: [[TMP57:%.*]] = call [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] @_cont_GetTriangleHitAttributes(ptr [[SYSTEM_DATA_ALLOCA]]) +; CLEANUP-NEXT: [[DOTFCA_0_EXTRACT52:%.*]] = extractvalue [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP57]], 0 ; CLEANUP-NEXT: [[DOTSROA_054_0_VEC_EXTRACT:%.*]] = extractelement <2 x float> [[DOTFCA_0_EXTRACT52]], i32 0 -; CLEANUP-NEXT: [[TMP55:%.*]] = bitcast float [[DOTSROA_054_0_VEC_EXTRACT]] to i32 +; CLEANUP-NEXT: [[TMP58:%.*]] = bitcast float [[DOTSROA_054_0_VEC_EXTRACT]] to i32 ; CLEANUP-NEXT: [[DOTSROA_054_4_VEC_EXTRACT:%.*]] = extractelement <2 x float> [[DOTFCA_0_EXTRACT52]], i32 1 -; CLEANUP-NEXT: [[TMP56:%.*]] = bitcast float [[DOTSROA_054_4_VEC_EXTRACT]] to i32 -; CLEANUP-NEXT: [[TMP57:%.*]] = load [[DX_TYPES_HANDLE:%.*]], ptr @"\01?Scene@@3URaytracingAccelerationStructure@@A", align 4 -; CLEANUP-NEXT: [[TMP58:%.*]] = load [[DX_TYPES_HANDLE]], ptr @"\01?RenderTarget@@3V?$RWTexture2D@V?$vector@M$03@@@@A", align 4 -; CLEANUP-NEXT: [[TMP59:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP57]]) -; CLEANUP-NEXT: [[TMP60:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP59]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) -; CLEANUP-NEXT: [[TMP61:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[TMP60]]) -; CLEANUP-NEXT: [[TMP62:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; CLEANUP-NEXT: [[DIS_DATA_I_FCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP62]], i32 0, i32 0 +; CLEANUP-NEXT: [[TMP59:%.*]] = bitcast float [[DOTSROA_054_4_VEC_EXTRACT]] to i32 +; CLEANUP-NEXT: [[TMP60:%.*]] = load [[DX_TYPES_HANDLE:%.*]], ptr @"\01?Scene@@3URaytracingAccelerationStructure@@A", align 4 +; CLEANUP-NEXT: [[TMP61:%.*]] = load [[DX_TYPES_HANDLE]], ptr @"\01?RenderTarget@@3V?$RWTexture2D@V?$vector@M$03@@@@A", align 4 +; CLEANUP-NEXT: [[TMP62:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP60]]) +; CLEANUP-NEXT: [[TMP63:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP62]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) +; CLEANUP-NEXT: [[TMP64:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[TMP63]]) +; CLEANUP-NEXT: [[TMP65:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; CLEANUP-NEXT: [[DIS_DATA_I_FCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP65]], i32 0, i32 0 ; CLEANUP-NEXT: [[DIS_DATA_I_FCA_0_LOAD:%.*]] = load i32, ptr [[DIS_DATA_I_FCA_0_GEP]], align 4 ; CLEANUP-NEXT: [[DIS_DATA_I_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, i32 [[DIS_DATA_I_FCA_0_LOAD]], 0 ; CLEANUP-NEXT: [[SYS_DATA_I:%.*]] = insertvalue [[STRUCT_SYSTEMDATA]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I_FCA_0_INSERT]], 0 ; CLEANUP-NEXT: [[TRAV_DATA_I:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA_I]], 0 -; CLEANUP-NEXT: [[TMP63:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: [[TMP64:%.*]] = load i32, ptr [[TMP63]], align 4 -; CLEANUP-NEXT: [[TMP65:%.*]] = add i32 [[TMP64]], 108 -; CLEANUP-NEXT: store i32 [[TMP65]], ptr [[TMP63]], align 4 -; CLEANUP-NEXT: store i32 [[TMP64]], ptr @PAYLOAD, align 4 -; CLEANUP-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; CLEANUP-NEXT: store i32 [[TMP3]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; CLEANUP-NEXT: store i32 [[TMP4]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; CLEANUP-NEXT: store i32 [[TMP5]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; CLEANUP-NEXT: store i32 [[TMP6]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 10), align 4 -; CLEANUP-NEXT: store i32 [[TMP7]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 11), align 4 -; CLEANUP-NEXT: store i32 [[TMP8]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 12), align 4 -; CLEANUP-NEXT: store i32 [[TMP9]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 13), align 4 -; CLEANUP-NEXT: store i32 [[TMP10]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 14), align 4 -; CLEANUP-NEXT: store i32 [[TMP11]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 15), align 4 -; CLEANUP-NEXT: store i32 [[TMP12]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 16), align 4 -; CLEANUP-NEXT: store i32 [[TMP13]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 17), align 4 -; CLEANUP-NEXT: store i32 [[TMP14]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 18), align 4 -; CLEANUP-NEXT: store i32 [[TMP15]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 19), align 4 -; CLEANUP-NEXT: store i32 [[TMP16]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 20), align 4 -; CLEANUP-NEXT: store i32 [[TMP17]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 21), align 4 -; CLEANUP-NEXT: store i32 [[TMP18]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 22), align 4 -; CLEANUP-NEXT: store i32 [[TMP19]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 23), align 4 -; CLEANUP-NEXT: store i32 [[TMP20]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 24), align 4 -; CLEANUP-NEXT: store i32 [[TMP21]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 25), align 4 -; CLEANUP-NEXT: store i32 [[TMP22]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 26), align 4 -; CLEANUP-NEXT: store i32 [[TMP23]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 27), align 4 -; CLEANUP-NEXT: store i32 [[TMP24]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 28), align 4 -; CLEANUP-NEXT: store i32 [[TMP25]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 29), align 4 -; CLEANUP-NEXT: store i32 [[TMP26]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 30), align 4 -; CLEANUP-NEXT: store i32 [[TMP27]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 31), align 4 -; CLEANUP-NEXT: store i32 [[TMP28]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 32), align 4 -; CLEANUP-NEXT: store i32 [[TMP29]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 33), align 4 -; CLEANUP-NEXT: store i32 [[TMP30]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 34), align 4 -; CLEANUP-NEXT: store i32 [[TMP31]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 35), align 4 -; CLEANUP-NEXT: store i32 [[TMP32]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 36), align 4 -; CLEANUP-NEXT: store i32 [[TMP33]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 37), align 4 -; CLEANUP-NEXT: store i32 [[TMP34]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 38), align 4 -; CLEANUP-NEXT: store i32 [[TMP35]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 39), align 4 -; CLEANUP-NEXT: store i32 [[TMP36]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 40), align 4 -; CLEANUP-NEXT: store i32 [[TMP37]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 41), align 4 -; CLEANUP-NEXT: store i32 [[TMP38]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 42), align 4 -; CLEANUP-NEXT: store i32 [[TMP39]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 43), align 4 -; CLEANUP-NEXT: store i32 [[TMP40]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 44), align 4 -; CLEANUP-NEXT: store i32 [[TMP41]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 45), align 4 -; CLEANUP-NEXT: store i32 [[TMP42]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 46), align 4 -; CLEANUP-NEXT: store i32 [[TMP43]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 47), align 4 -; CLEANUP-NEXT: store i32 [[TMP44]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 48), align 4 -; CLEANUP-NEXT: store i32 [[TMP45]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 49), align 4 -; CLEANUP-NEXT: store i32 [[TMP46]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 50), align 4 -; CLEANUP-NEXT: store i32 [[TMP47]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 51), align 4 -; CLEANUP-NEXT: store i32 [[TMP48]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 52), align 4 -; CLEANUP-NEXT: store i32 [[TMP49]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 53), align 4 -; CLEANUP-NEXT: store i32 [[TMP50]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 54), align 4 -; CLEANUP-NEXT: store i32 [[TMP51]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 55), align 4 -; CLEANUP-NEXT: store i32 [[TMP52]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 56), align 4 ; CLEANUP-NEXT: [[TMP66:%.*]] = call ptr @continuation.getContinuationStackOffset() ; CLEANUP-NEXT: [[TMP67:%.*]] = load i32, ptr [[TMP66]], align 4 -; CLEANUP-NEXT: [[TMP68:%.*]] = add i32 [[TMP67]], 16 -; CLEANUP-NEXT: store i32 [[TMP68]], ptr [[TMP66]], align 4 +; CLEANUP-NEXT: [[TMP68:%.*]] = add i32 [[TMP67]], -108 +; CLEANUP-NEXT: store i32 [[TMP68]], ptr @PAYLOAD, align 4 +; CLEANUP-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) +; CLEANUP-NEXT: store i32 [[TMP6]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; CLEANUP-NEXT: store i32 [[TMP7]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; CLEANUP-NEXT: store i32 [[TMP8]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; CLEANUP-NEXT: store i32 [[TMP9]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 10), align 4 +; CLEANUP-NEXT: store i32 [[TMP10]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 11), align 4 +; CLEANUP-NEXT: store i32 [[TMP11]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 12), align 4 +; CLEANUP-NEXT: store i32 [[TMP12]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 13), align 4 +; CLEANUP-NEXT: store i32 [[TMP13]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 14), align 4 +; CLEANUP-NEXT: store i32 [[TMP14]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 15), align 4 +; CLEANUP-NEXT: store i32 [[TMP15]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 16), align 4 +; CLEANUP-NEXT: store i32 [[TMP16]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 17), align 4 +; CLEANUP-NEXT: store i32 [[TMP17]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 18), align 4 +; CLEANUP-NEXT: store i32 [[TMP18]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 19), align 4 +; CLEANUP-NEXT: store i32 [[TMP19]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 20), align 4 +; CLEANUP-NEXT: store i32 [[TMP20]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 21), align 4 +; CLEANUP-NEXT: store i32 [[TMP21]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 22), align 4 +; CLEANUP-NEXT: store i32 [[TMP22]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 23), align 4 +; CLEANUP-NEXT: store i32 [[TMP23]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 24), align 4 +; CLEANUP-NEXT: store i32 [[TMP24]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 25), align 4 +; CLEANUP-NEXT: store i32 [[TMP25]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 26), align 4 +; CLEANUP-NEXT: store i32 [[TMP26]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 27), align 4 +; CLEANUP-NEXT: store i32 [[TMP27]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 28), align 4 +; CLEANUP-NEXT: store i32 [[TMP28]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 29), align 4 +; CLEANUP-NEXT: store i32 [[TMP29]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 30), align 4 +; CLEANUP-NEXT: store i32 [[TMP30]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 31), align 4 +; CLEANUP-NEXT: store i32 [[TMP31]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 32), align 4 +; CLEANUP-NEXT: store i32 [[TMP32]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 33), align 4 +; CLEANUP-NEXT: store i32 [[TMP33]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 34), align 4 +; CLEANUP-NEXT: store i32 [[TMP34]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 35), align 4 +; CLEANUP-NEXT: store i32 [[TMP35]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 36), align 4 +; CLEANUP-NEXT: store i32 [[TMP36]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 37), align 4 +; CLEANUP-NEXT: store i32 [[TMP37]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 38), align 4 +; CLEANUP-NEXT: store i32 [[TMP38]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 39), align 4 +; CLEANUP-NEXT: store i32 [[TMP39]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 40), align 4 +; CLEANUP-NEXT: store i32 [[TMP40]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 41), align 4 +; CLEANUP-NEXT: store i32 [[TMP41]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 42), align 4 +; CLEANUP-NEXT: store i32 [[TMP42]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 43), align 4 +; CLEANUP-NEXT: store i32 [[TMP43]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 44), align 4 +; CLEANUP-NEXT: store i32 [[TMP44]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 45), align 4 +; CLEANUP-NEXT: store i32 [[TMP45]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 46), align 4 +; CLEANUP-NEXT: store i32 [[TMP46]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 47), align 4 +; CLEANUP-NEXT: store i32 [[TMP47]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 48), align 4 +; CLEANUP-NEXT: store i32 [[TMP48]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 49), align 4 +; CLEANUP-NEXT: store i32 [[TMP49]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 50), align 4 +; CLEANUP-NEXT: store i32 [[TMP50]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 51), align 4 +; CLEANUP-NEXT: store i32 [[TMP51]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 52), align 4 +; CLEANUP-NEXT: store i32 [[TMP52]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 53), align 4 +; CLEANUP-NEXT: store i32 [[TMP53]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 54), align 4 +; CLEANUP-NEXT: store i32 [[TMP54]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 55), align 4 +; CLEANUP-NEXT: store i32 [[TMP55]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 56), align 4 ; CLEANUP-NEXT: [[TMP69:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP69]]) -; CLEANUP-NEXT: [[TMP70:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CLEANUP-NEXT: [[TMP71:%.*]] = load i32, ptr [[TMP70]], align 4 -; CLEANUP-NEXT: store i32 [[TMP71]], ptr @CONTINUATION_STATE, align 4 -; CLEANUP-NEXT: [[TMP72:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CLEANUP-NEXT: [[TMP73:%.*]] = load i32, ptr [[TMP72]], align 4 -; CLEANUP-NEXT: store i32 [[TMP73]], ptr getelementptr inbounds ([4 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CLEANUP-NEXT: [[TMP74:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 2 -; CLEANUP-NEXT: [[TMP75:%.*]] = load i32, ptr [[TMP74]], align 4 -; CLEANUP-NEXT: store i32 [[TMP75]], ptr getelementptr inbounds ([4 x i32], ptr @CONTINUATION_STATE, i32 0, i32 2), align 4 -; CLEANUP-NEXT: [[TMP76:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 3 -; CLEANUP-NEXT: [[TMP77:%.*]] = load i32, ptr [[TMP76]], align 4 -; CLEANUP-NEXT: store i32 [[TMP77]], ptr getelementptr inbounds ([4 x i32], ptr @CONTINUATION_STATE, i32 0, i32 3), align 4 -; CLEANUP-NEXT: [[TMP78:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: [[TMP79:%.*]] = load i32, ptr [[TMP78]], align 4 -; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 4, i32 [[TMP79]], i64 ptrtoint (ptr @ClosestHit.resume.0 to i64), [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]]), !continuation.registercount !19, !continuation.returnedRegistercount !19 +; CLEANUP-NEXT: [[TMP70:%.*]] = load i32, ptr [[TMP69]], align 4 +; CLEANUP-NEXT: [[TMP71:%.*]] = add i32 [[TMP70]], 16 +; CLEANUP-NEXT: store i32 [[TMP71]], ptr [[TMP69]], align 4 +; CLEANUP-NEXT: [[TMP72:%.*]] = inttoptr i32 [[TMP70]] to ptr addrspace(21) +; CLEANUP-NEXT: [[TMP73:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; CLEANUP-NEXT: [[TMP74:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(21) [[TMP72]], i32 0, i32 0 +; CLEANUP-NEXT: [[TMP75:%.*]] = load i32, ptr [[TMP73]], align 4 +; CLEANUP-NEXT: store i32 [[TMP75]], ptr addrspace(21) [[TMP74]], align 4 +; CLEANUP-NEXT: [[TMP76:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CLEANUP-NEXT: [[TMP77:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(21) [[TMP72]], i32 0, i32 1 +; CLEANUP-NEXT: [[TMP78:%.*]] = load i32, ptr [[TMP76]], align 4 +; CLEANUP-NEXT: store i32 [[TMP78]], ptr addrspace(21) [[TMP77]], align 4 +; CLEANUP-NEXT: [[TMP79:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; CLEANUP-NEXT: [[TMP80:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(21) [[TMP72]], i32 0, i32 2 +; CLEANUP-NEXT: [[TMP81:%.*]] = load i32, ptr [[TMP79]], align 4 +; CLEANUP-NEXT: store i32 [[TMP81]], ptr addrspace(21) [[TMP80]], align 4 +; CLEANUP-NEXT: [[TMP82:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 3 +; CLEANUP-NEXT: [[TMP83:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(21) [[TMP72]], i32 0, i32 3 +; CLEANUP-NEXT: [[TMP84:%.*]] = load i32, ptr [[TMP82]], align 4 +; CLEANUP-NEXT: store i32 [[TMP84]], ptr addrspace(21) [[TMP83]], align 4 +; CLEANUP-NEXT: [[TMP85:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP86:%.*]] = load i32, ptr [[TMP85]], align 4 +; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 4, i32 [[TMP86]], i64 ptrtoint (ptr @ClosestHit.resume.0 to i64), [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]]), !continuation.registercount !18, !continuation.returnedRegistercount !18 ; CLEANUP-NEXT: unreachable ; ; -; CLEANUP-LABEL: define void @ClosestHit.resume.0( -; CLEANUP-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage !26 !continuation.registercount !19 !continuation !27 { +; CLEANUP-LABEL: define dso_local void @ClosestHit.resume.0( +; CLEANUP-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage !25 !continuation.registercount !18 !continuation !26 { ; CLEANUP-NEXT: entryresume.0: ; CLEANUP-NEXT: [[CONT_STATE:%.*]] = alloca [4 x i32], align 4 -; CLEANUP-NEXT: [[TMP2:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; CLEANUP-NEXT: [[TMP3:%.*]] = load i32, ptr @CONTINUATION_STATE, align 4 -; CLEANUP-NEXT: store i32 [[TMP3]], ptr [[TMP2]], align 4 -; CLEANUP-NEXT: [[TMP4:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; CLEANUP-NEXT: [[TMP5:%.*]] = load i32, ptr getelementptr inbounds ([4 x i32], ptr @CONTINUATION_STATE, i32 0, i32 1), align 4 -; CLEANUP-NEXT: store i32 [[TMP5]], ptr [[TMP4]], align 4 -; CLEANUP-NEXT: [[TMP6:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 2 -; CLEANUP-NEXT: [[TMP7:%.*]] = load i32, ptr getelementptr inbounds ([4 x i32], ptr @CONTINUATION_STATE, i32 0, i32 2), align 4 -; CLEANUP-NEXT: store i32 [[TMP7]], ptr [[TMP6]], align 4 -; CLEANUP-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 3 -; CLEANUP-NEXT: [[TMP9:%.*]] = load i32, ptr getelementptr inbounds ([4 x i32], ptr @CONTINUATION_STATE, i32 0, i32 3), align 4 -; CLEANUP-NEXT: store i32 [[TMP9]], ptr [[TMP8]], align 4 -; CLEANUP-NEXT: [[TMP10:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @CONTINUATION_STATE, ptr [[TMP10]]) -; CLEANUP-NEXT: [[TMP11:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4 -; CLEANUP-NEXT: [[TMP13:%.*]] = add i32 [[TMP12]], -16 -; CLEANUP-NEXT: store i32 [[TMP13]], ptr [[TMP11]], align 4 +; CLEANUP-NEXT: [[TMP2:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4 +; CLEANUP-NEXT: [[TMP4:%.*]] = inttoptr i32 [[TMP3]] to ptr addrspace(21) +; CLEANUP-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP4]], i64 -16 +; CLEANUP-NEXT: [[TMP6:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(21) [[TMP5]], i32 0, i32 0 +; CLEANUP-NEXT: [[TMP7:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; CLEANUP-NEXT: [[TMP8:%.*]] = load i32, ptr addrspace(21) [[TMP6]], align 4 +; CLEANUP-NEXT: store i32 [[TMP8]], ptr [[TMP7]], align 4 +; CLEANUP-NEXT: [[TMP9:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(21) [[TMP5]], i32 0, i32 1 +; CLEANUP-NEXT: [[TMP10:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; CLEANUP-NEXT: [[TMP11:%.*]] = load i32, ptr addrspace(21) [[TMP9]], align 4 +; CLEANUP-NEXT: store i32 [[TMP11]], ptr [[TMP10]], align 4 +; CLEANUP-NEXT: [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(21) [[TMP5]], i32 0, i32 2 +; CLEANUP-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; CLEANUP-NEXT: [[TMP14:%.*]] = load i32, ptr addrspace(21) [[TMP12]], align 4 +; CLEANUP-NEXT: store i32 [[TMP14]], ptr [[TMP13]], align 4 +; CLEANUP-NEXT: [[TMP15:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(21) [[TMP5]], i32 0, i32 3 +; CLEANUP-NEXT: [[TMP16:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 3 +; CLEANUP-NEXT: [[TMP17:%.*]] = load i32, ptr addrspace(21) [[TMP15]], align 4 +; CLEANUP-NEXT: store i32 [[TMP17]], ptr [[TMP16]], align 4 +; CLEANUP-NEXT: [[TMP18:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP18]], align 4 +; CLEANUP-NEXT: [[TMP20:%.*]] = add i32 [[TMP19]], -16 +; CLEANUP-NEXT: store i32 [[TMP20]], ptr [[TMP18]], align 4 ; CLEANUP-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = getelementptr inbounds [[CLOSESTHIT_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 -; CLEANUP-NEXT: [[TMP14:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA:%.*]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; CLEANUP-NEXT: [[TMP15:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; CLEANUP-NEXT: [[TMP16:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; CLEANUP-NEXT: [[TMP17:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; CLEANUP-NEXT: [[TMP18:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 10), align 4 -; CLEANUP-NEXT: [[TMP19:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 11), align 4 -; CLEANUP-NEXT: [[TMP20:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 12), align 4 -; CLEANUP-NEXT: [[TMP21:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 13), align 4 -; CLEANUP-NEXT: [[TMP22:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 14), align 4 -; CLEANUP-NEXT: [[TMP23:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 15), align 4 -; CLEANUP-NEXT: [[TMP24:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 16), align 4 -; CLEANUP-NEXT: [[TMP25:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 17), align 4 -; CLEANUP-NEXT: [[TMP26:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 18), align 4 -; CLEANUP-NEXT: [[TMP27:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 19), align 4 -; CLEANUP-NEXT: [[TMP28:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 20), align 4 -; CLEANUP-NEXT: [[TMP29:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 21), align 4 -; CLEANUP-NEXT: [[TMP30:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 22), align 4 -; CLEANUP-NEXT: [[TMP31:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 23), align 4 -; CLEANUP-NEXT: [[TMP32:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 24), align 4 -; CLEANUP-NEXT: [[TMP33:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 25), align 4 -; CLEANUP-NEXT: [[TMP34:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 26), align 4 -; CLEANUP-NEXT: [[TMP35:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 27), align 4 -; CLEANUP-NEXT: [[TMP36:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 28), align 4 -; CLEANUP-NEXT: [[TMP37:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 29), align 4 -; CLEANUP-NEXT: [[TMP38:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 30), align 4 -; CLEANUP-NEXT: [[TMP39:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 31), align 4 -; CLEANUP-NEXT: [[TMP40:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 32), align 4 -; CLEANUP-NEXT: [[TMP41:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 33), align 4 -; CLEANUP-NEXT: [[TMP42:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 34), align 4 -; CLEANUP-NEXT: [[TMP43:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 35), align 4 -; CLEANUP-NEXT: [[TMP44:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 36), align 4 -; CLEANUP-NEXT: [[TMP45:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 37), align 4 -; CLEANUP-NEXT: [[TMP46:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 38), align 4 -; CLEANUP-NEXT: [[TMP47:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 39), align 4 -; CLEANUP-NEXT: [[TMP48:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 40), align 4 -; CLEANUP-NEXT: [[TMP49:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 41), align 4 -; CLEANUP-NEXT: [[TMP50:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 42), align 4 -; CLEANUP-NEXT: [[TMP51:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 43), align 4 -; CLEANUP-NEXT: [[TMP52:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 44), align 4 -; CLEANUP-NEXT: [[TMP53:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 45), align 4 -; CLEANUP-NEXT: [[TMP54:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 46), align 4 -; CLEANUP-NEXT: [[TMP55:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 47), align 4 -; CLEANUP-NEXT: [[TMP56:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 48), align 4 -; CLEANUP-NEXT: [[TMP57:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 49), align 4 -; CLEANUP-NEXT: [[TMP58:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 50), align 4 -; CLEANUP-NEXT: [[TMP59:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 51), align 4 -; CLEANUP-NEXT: [[TMP60:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 52), align 4 -; CLEANUP-NEXT: [[TMP61:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 53), align 4 -; CLEANUP-NEXT: [[TMP62:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 54), align 4 -; CLEANUP-NEXT: [[TMP63:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 55), align 4 -; CLEANUP-NEXT: [[TMP64:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 56), align 4 -; CLEANUP-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; CLEANUP-NEXT: [[TMP65:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: [[TMP66:%.*]] = load i32, ptr [[TMP65]], align 4 -; CLEANUP-NEXT: [[TMP67:%.*]] = add i32 [[TMP66]], -108 -; CLEANUP-NEXT: store i32 [[TMP67]], ptr [[TMP65]], align 4 +; CLEANUP-NEXT: [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA:%.*]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; CLEANUP-NEXT: [[TMP22:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; CLEANUP-NEXT: [[TMP23:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; CLEANUP-NEXT: [[TMP24:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; CLEANUP-NEXT: [[TMP25:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 10), align 4 +; CLEANUP-NEXT: [[TMP26:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 11), align 4 +; CLEANUP-NEXT: [[TMP27:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 12), align 4 +; CLEANUP-NEXT: [[TMP28:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 13), align 4 +; CLEANUP-NEXT: [[TMP29:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 14), align 4 +; CLEANUP-NEXT: [[TMP30:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 15), align 4 +; CLEANUP-NEXT: [[TMP31:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 16), align 4 +; CLEANUP-NEXT: [[TMP32:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 17), align 4 +; CLEANUP-NEXT: [[TMP33:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 18), align 4 +; CLEANUP-NEXT: [[TMP34:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 19), align 4 +; CLEANUP-NEXT: [[TMP35:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 20), align 4 +; CLEANUP-NEXT: [[TMP36:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 21), align 4 +; CLEANUP-NEXT: [[TMP37:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 22), align 4 +; CLEANUP-NEXT: [[TMP38:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 23), align 4 +; CLEANUP-NEXT: [[TMP39:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 24), align 4 +; CLEANUP-NEXT: [[TMP40:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 25), align 4 +; CLEANUP-NEXT: [[TMP41:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 26), align 4 +; CLEANUP-NEXT: [[TMP42:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 27), align 4 +; CLEANUP-NEXT: [[TMP43:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 28), align 4 +; CLEANUP-NEXT: [[TMP44:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 29), align 4 +; CLEANUP-NEXT: [[TMP45:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 30), align 4 +; CLEANUP-NEXT: [[TMP46:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 31), align 4 +; CLEANUP-NEXT: [[TMP47:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 32), align 4 +; CLEANUP-NEXT: [[TMP48:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 33), align 4 +; CLEANUP-NEXT: [[TMP49:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 34), align 4 +; CLEANUP-NEXT: [[TMP50:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 35), align 4 +; CLEANUP-NEXT: [[TMP51:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 36), align 4 +; CLEANUP-NEXT: [[TMP52:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 37), align 4 +; CLEANUP-NEXT: [[TMP53:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 38), align 4 +; CLEANUP-NEXT: [[TMP54:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 39), align 4 +; CLEANUP-NEXT: [[TMP55:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 40), align 4 +; CLEANUP-NEXT: [[TMP56:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 41), align 4 +; CLEANUP-NEXT: [[TMP57:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 42), align 4 +; CLEANUP-NEXT: [[TMP58:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 43), align 4 +; CLEANUP-NEXT: [[TMP59:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 44), align 4 +; CLEANUP-NEXT: [[TMP60:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 45), align 4 +; CLEANUP-NEXT: [[TMP61:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 46), align 4 +; CLEANUP-NEXT: [[TMP62:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 47), align 4 +; CLEANUP-NEXT: [[TMP63:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 48), align 4 +; CLEANUP-NEXT: [[TMP64:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 49), align 4 +; CLEANUP-NEXT: [[TMP65:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 50), align 4 +; CLEANUP-NEXT: [[TMP66:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 51), align 4 +; CLEANUP-NEXT: [[TMP67:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 52), align 4 +; CLEANUP-NEXT: [[TMP68:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 53), align 4 +; CLEANUP-NEXT: [[TMP69:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 54), align 4 +; CLEANUP-NEXT: [[TMP70:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 55), align 4 +; CLEANUP-NEXT: [[TMP71:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 56), align 4 ; CLEANUP-NEXT: [[DOTFCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], 0 -; CLEANUP-NEXT: [[DOTFCA_0_GEP51:%.*]] = getelementptr inbounds [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[TMP14]], i32 0, i32 0 +; CLEANUP-NEXT: [[DOTFCA_0_GEP51:%.*]] = getelementptr inbounds [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[TMP21]], i32 0, i32 0 ; CLEANUP-NEXT: store i32 [[DOTFCA_0_EXTRACT]], ptr [[DOTFCA_0_GEP51]], align 4 ; CLEANUP-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; CLEANUP-NEXT: [[DOTRELOAD_ADDR:%.*]] = getelementptr inbounds [[CLOSESTHIT_FRAME]], ptr [[CONT_STATE]], i32 0, i32 2 @@ -727,64 +735,67 @@ attributes #3 = { nounwind } ; CLEANUP-NEXT: [[RETURNADDR_RELOAD:%.*]] = load i64, ptr [[RETURNADDR_RELOAD_ADDR]], align 4 ; CLEANUP-NEXT: store i32 [[DOTRELOAD]], ptr @PAYLOAD, align 4 ; CLEANUP-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; CLEANUP-NEXT: store i32 [[TMP15]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; CLEANUP-NEXT: store i32 [[TMP16]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; CLEANUP-NEXT: store i32 [[TMP17]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; CLEANUP-NEXT: store i32 [[TMP18]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 10), align 4 -; CLEANUP-NEXT: store i32 [[TMP19]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 11), align 4 -; CLEANUP-NEXT: store i32 [[TMP20]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 12), align 4 -; CLEANUP-NEXT: store i32 [[TMP21]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 13), align 4 -; CLEANUP-NEXT: store i32 [[TMP22]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 14), align 4 -; CLEANUP-NEXT: store i32 [[TMP23]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 15), align 4 -; CLEANUP-NEXT: store i32 [[TMP24]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 16), align 4 -; CLEANUP-NEXT: store i32 [[TMP25]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 17), align 4 -; CLEANUP-NEXT: store i32 [[TMP26]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 18), align 4 -; CLEANUP-NEXT: store i32 [[TMP27]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 19), align 4 -; CLEANUP-NEXT: store i32 [[TMP28]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 20), align 4 -; CLEANUP-NEXT: store i32 [[TMP29]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 21), align 4 -; CLEANUP-NEXT: store i32 [[TMP30]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 22), align 4 -; CLEANUP-NEXT: store i32 [[TMP31]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 23), align 4 -; CLEANUP-NEXT: store i32 [[TMP32]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 24), align 4 -; CLEANUP-NEXT: store i32 [[TMP33]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 25), align 4 -; CLEANUP-NEXT: store i32 [[TMP34]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 26), align 4 -; CLEANUP-NEXT: store i32 [[TMP35]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 27), align 4 -; CLEANUP-NEXT: store i32 [[TMP36]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 28), align 4 -; CLEANUP-NEXT: store i32 [[TMP37]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 29), align 4 -; CLEANUP-NEXT: store i32 [[TMP38]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 30), align 4 -; CLEANUP-NEXT: store i32 [[TMP39]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 31), align 4 -; CLEANUP-NEXT: store i32 [[TMP40]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 32), align 4 -; CLEANUP-NEXT: store i32 [[TMP41]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 33), align 4 -; CLEANUP-NEXT: store i32 [[TMP42]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 34), align 4 -; CLEANUP-NEXT: store i32 [[TMP43]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 35), align 4 -; CLEANUP-NEXT: store i32 [[TMP44]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 36), align 4 -; CLEANUP-NEXT: store i32 [[TMP45]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 37), align 4 -; CLEANUP-NEXT: store i32 [[TMP46]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 38), align 4 -; CLEANUP-NEXT: store i32 [[TMP47]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 39), align 4 -; CLEANUP-NEXT: store i32 [[TMP48]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 40), align 4 -; CLEANUP-NEXT: store i32 [[TMP49]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 41), align 4 -; CLEANUP-NEXT: store i32 [[TMP50]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 42), align 4 -; CLEANUP-NEXT: store i32 [[TMP51]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 43), align 4 -; CLEANUP-NEXT: store i32 [[TMP52]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 44), align 4 -; CLEANUP-NEXT: store i32 [[TMP53]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 45), align 4 -; CLEANUP-NEXT: store i32 [[TMP54]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 46), align 4 -; CLEANUP-NEXT: store i32 [[TMP55]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 47), align 4 -; CLEANUP-NEXT: store i32 [[TMP56]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 48), align 4 -; CLEANUP-NEXT: store i32 [[TMP57]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 49), align 4 -; CLEANUP-NEXT: store i32 [[TMP58]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 50), align 4 -; CLEANUP-NEXT: store i32 [[TMP59]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 51), align 4 -; CLEANUP-NEXT: store i32 [[TMP60]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 52), align 4 -; CLEANUP-NEXT: store i32 [[TMP61]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 53), align 4 -; CLEANUP-NEXT: store i32 [[TMP62]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 54), align 4 -; CLEANUP-NEXT: store i32 [[TMP63]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 55), align 4 -; CLEANUP-NEXT: store i32 [[TMP64]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 56), align 4 -; CLEANUP-NEXT: [[TMP68:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; CLEANUP-NEXT: [[DOTFCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[TMP68]], i32 0, i32 0 +; CLEANUP-NEXT: store i32 [[TMP22]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; CLEANUP-NEXT: store i32 [[TMP23]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; CLEANUP-NEXT: store i32 [[TMP24]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; CLEANUP-NEXT: store i32 [[TMP25]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 10), align 4 +; CLEANUP-NEXT: store i32 [[TMP26]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 11), align 4 +; CLEANUP-NEXT: store i32 [[TMP27]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 12), align 4 +; CLEANUP-NEXT: store i32 [[TMP28]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 13), align 4 +; CLEANUP-NEXT: store i32 [[TMP29]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 14), align 4 +; CLEANUP-NEXT: store i32 [[TMP30]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 15), align 4 +; CLEANUP-NEXT: store i32 [[TMP31]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 16), align 4 +; CLEANUP-NEXT: store i32 [[TMP32]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 17), align 4 +; CLEANUP-NEXT: store i32 [[TMP33]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 18), align 4 +; CLEANUP-NEXT: store i32 [[TMP34]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 19), align 4 +; CLEANUP-NEXT: store i32 [[TMP35]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 20), align 4 +; CLEANUP-NEXT: store i32 [[TMP36]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 21), align 4 +; CLEANUP-NEXT: store i32 [[TMP37]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 22), align 4 +; CLEANUP-NEXT: store i32 [[TMP38]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 23), align 4 +; CLEANUP-NEXT: store i32 [[TMP39]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 24), align 4 +; CLEANUP-NEXT: store i32 [[TMP40]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 25), align 4 +; CLEANUP-NEXT: store i32 [[TMP41]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 26), align 4 +; CLEANUP-NEXT: store i32 [[TMP42]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 27), align 4 +; CLEANUP-NEXT: store i32 [[TMP43]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 28), align 4 +; CLEANUP-NEXT: store i32 [[TMP44]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 29), align 4 +; CLEANUP-NEXT: store i32 [[TMP45]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 30), align 4 +; CLEANUP-NEXT: store i32 [[TMP46]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 31), align 4 +; CLEANUP-NEXT: store i32 [[TMP47]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 32), align 4 +; CLEANUP-NEXT: store i32 [[TMP48]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 33), align 4 +; CLEANUP-NEXT: store i32 [[TMP49]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 34), align 4 +; CLEANUP-NEXT: store i32 [[TMP50]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 35), align 4 +; CLEANUP-NEXT: store i32 [[TMP51]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 36), align 4 +; CLEANUP-NEXT: store i32 [[TMP52]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 37), align 4 +; CLEANUP-NEXT: store i32 [[TMP53]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 38), align 4 +; CLEANUP-NEXT: store i32 [[TMP54]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 39), align 4 +; CLEANUP-NEXT: store i32 [[TMP55]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 40), align 4 +; CLEANUP-NEXT: store i32 [[TMP56]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 41), align 4 +; CLEANUP-NEXT: store i32 [[TMP57]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 42), align 4 +; CLEANUP-NEXT: store i32 [[TMP58]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 43), align 4 +; CLEANUP-NEXT: store i32 [[TMP59]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 44), align 4 +; CLEANUP-NEXT: store i32 [[TMP60]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 45), align 4 +; CLEANUP-NEXT: store i32 [[TMP61]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 46), align 4 +; CLEANUP-NEXT: store i32 [[TMP62]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 47), align 4 +; CLEANUP-NEXT: store i32 [[TMP63]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 48), align 4 +; CLEANUP-NEXT: store i32 [[TMP64]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 49), align 4 +; CLEANUP-NEXT: store i32 [[TMP65]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 50), align 4 +; CLEANUP-NEXT: store i32 [[TMP66]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 51), align 4 +; CLEANUP-NEXT: store i32 [[TMP67]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 52), align 4 +; CLEANUP-NEXT: store i32 [[TMP68]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 53), align 4 +; CLEANUP-NEXT: store i32 [[TMP69]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 54), align 4 +; CLEANUP-NEXT: store i32 [[TMP70]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 55), align 4 +; CLEANUP-NEXT: store i32 [[TMP71]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 56), align 4 +; CLEANUP-NEXT: [[TMP72:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP73:%.*]] = load i32, ptr [[TMP72]], align 4 +; CLEANUP-NEXT: [[TMP74:%.*]] = add i32 [[TMP73]], -108 +; CLEANUP-NEXT: store i32 [[TMP74]], ptr [[TMP72]], align 4 +; CLEANUP-NEXT: [[TMP75:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; CLEANUP-NEXT: [[DOTFCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[TMP75]], i32 0, i32 0 ; CLEANUP-NEXT: [[DOTFCA_0_LOAD:%.*]] = load i32, ptr [[DOTFCA_0_GEP]], align 4 ; CLEANUP-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, i32 [[DOTFCA_0_LOAD]], 0 -; CLEANUP-NEXT: call void @continuation.restore.continuation_state() -; CLEANUP-NEXT: [[TMP69:%.*]] = call ptr @continuation.getContinuationStackOffset() -; CLEANUP-NEXT: [[TMP70:%.*]] = load i32, ptr [[TMP69]], align 4 -; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP70]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount !19 +; CLEANUP-NEXT: [[TMP76:%.*]] = call ptr @continuation.getContinuationStackOffset() +; CLEANUP-NEXT: [[TMP77:%.*]] = load i32, ptr [[TMP76]], align 4 +; CLEANUP-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP77]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount !18 ; CLEANUP-NEXT: unreachable ; ; @@ -805,18 +816,20 @@ attributes #3 = { nounwind } ; POST-PROCESS-NEXT: [[TMP2:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA]], align 4 ; POST-PROCESS-NEXT: [[DOTFCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP2]], 0 ; POST-PROCESS-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) -; POST-PROCESS-NEXT: [[TMP3:%.*]] = load [[DX_TYPES_HANDLE:%.*]], ptr @"\01?Scene@@3URaytracingAccelerationStructure@@A", align 4 -; POST-PROCESS-NEXT: [[TMP4:%.*]] = load [[DX_TYPES_HANDLE]], ptr @"\01?RenderTarget@@3V?$RWTexture2D@V?$vector@M$03@@@@A", align 4 -; POST-PROCESS-NEXT: [[TMP5:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP3]]) -; POST-PROCESS-NEXT: [[TMP6:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP5]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) -; POST-PROCESS-NEXT: [[TMP7:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[TMP6]]) +; POST-PROCESS-NEXT: [[TMP3:%.*]] = load i32, ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 108 +; POST-PROCESS-NEXT: store i32 [[TMP4]], ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP5:%.*]] = load [[DX_TYPES_HANDLE:%.*]], ptr @"\01?Scene@@3URaytracingAccelerationStructure@@A", align 4 +; POST-PROCESS-NEXT: [[TMP6:%.*]] = load [[DX_TYPES_HANDLE]], ptr @"\01?RenderTarget@@3V?$RWTexture2D@V?$vector@M$03@@@@A", align 4 +; POST-PROCESS-NEXT: [[TMP7:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP5]]) +; POST-PROCESS-NEXT: [[TMP8:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP7]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) +; POST-PROCESS-NEXT: [[TMP9:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[TMP8]]) ; POST-PROCESS-NEXT: [[DIS_DATA_I_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, i32 [[DOTFCA_0_EXTRACT]], 0 ; POST-PROCESS-NEXT: [[SYS_DATA_I:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I_FCA_0_INSERT]], 0 ; POST-PROCESS-NEXT: [[TRAV_DATA_I:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA_I]], 0 -; POST-PROCESS-NEXT: [[TMP8:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP9:%.*]] = add i32 [[TMP8]], 108 -; POST-PROCESS-NEXT: store i32 [[TMP9]], ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: store i32 [[TMP8]], ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP10:%.*]] = load i32, ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP11:%.*]] = add i32 [[TMP10]], -108 +; POST-PROCESS-NEXT: store i32 [[TMP11]], ptr addrspace(20) @REGISTERS, align 4 ; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(20) addrspacecast (ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT:%.*]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i32 7) to ptr addrspace(20)), align 4 ; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 8) to ptr addrspace(20)), align 4 ; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 9) to ptr addrspace(20)), align 4 @@ -840,148 +853,148 @@ attributes #3 = { nounwind } ; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 27) to ptr addrspace(20)), align 4 ; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 28) to ptr addrspace(20)), align 4 ; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 29) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP10:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP11:%.*]] = inttoptr i32 [[TMP10]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP11]], i32 -30 -; POST-PROCESS-NEXT: [[TMP13:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP12]], i32 0, i32 0, i64 30 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP13]], align 4 -; POST-PROCESS-NEXT: [[TMP14:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP15:%.*]] = inttoptr i32 [[TMP14]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP15]], i32 -30 -; POST-PROCESS-NEXT: [[TMP17:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP16]], i32 0, i32 0, i64 31 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP17]], align 4 -; POST-PROCESS-NEXT: [[TMP18:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP19:%.*]] = inttoptr i32 [[TMP18]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP19]], i32 -30 -; POST-PROCESS-NEXT: [[TMP21:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP20]], i32 0, i32 0, i64 32 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP21]], align 4 -; POST-PROCESS-NEXT: [[TMP22:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP23:%.*]] = inttoptr i32 [[TMP22]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP24:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP23]], i32 -30 -; POST-PROCESS-NEXT: [[TMP25:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP24]], i32 0, i32 0, i64 33 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP25]], align 4 -; POST-PROCESS-NEXT: [[TMP26:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP27:%.*]] = inttoptr i32 [[TMP26]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP27]], i32 -30 -; POST-PROCESS-NEXT: [[TMP29:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP28]], i32 0, i32 0, i64 34 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP29]], align 4 -; POST-PROCESS-NEXT: [[TMP30:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP31:%.*]] = inttoptr i32 [[TMP30]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP31]], i32 -30 -; POST-PROCESS-NEXT: [[TMP33:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP32]], i32 0, i32 0, i64 35 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP33]], align 4 -; POST-PROCESS-NEXT: [[TMP34:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP35:%.*]] = inttoptr i32 [[TMP34]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP36:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP35]], i32 -30 -; POST-PROCESS-NEXT: [[TMP37:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP36]], i32 0, i32 0, i64 36 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP37]], align 4 -; POST-PROCESS-NEXT: [[TMP38:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP39:%.*]] = inttoptr i32 [[TMP38]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP40:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP39]], i32 -30 -; POST-PROCESS-NEXT: [[TMP41:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP40]], i32 0, i32 0, i64 37 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP41]], align 4 -; POST-PROCESS-NEXT: [[TMP42:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP43:%.*]] = inttoptr i32 [[TMP42]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP44:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP43]], i32 -30 -; POST-PROCESS-NEXT: [[TMP45:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP44]], i32 0, i32 0, i64 38 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP45]], align 4 -; POST-PROCESS-NEXT: [[TMP46:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP47:%.*]] = inttoptr i32 [[TMP46]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP48:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP47]], i32 -30 -; POST-PROCESS-NEXT: [[TMP49:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP48]], i32 0, i32 0, i64 39 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP49]], align 4 -; POST-PROCESS-NEXT: [[TMP50:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP51:%.*]] = inttoptr i32 [[TMP50]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP52:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP51]], i32 -30 -; POST-PROCESS-NEXT: [[TMP53:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP52]], i32 0, i32 0, i64 40 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP53]], align 4 -; POST-PROCESS-NEXT: [[TMP54:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP55:%.*]] = inttoptr i32 [[TMP54]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP56:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP55]], i32 -30 -; POST-PROCESS-NEXT: [[TMP57:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP56]], i32 0, i32 0, i64 41 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP57]], align 4 -; POST-PROCESS-NEXT: [[TMP58:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP59:%.*]] = inttoptr i32 [[TMP58]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP60:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP59]], i32 -30 -; POST-PROCESS-NEXT: [[TMP61:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP60]], i32 0, i32 0, i64 42 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP61]], align 4 -; POST-PROCESS-NEXT: [[TMP62:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP63:%.*]] = inttoptr i32 [[TMP62]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP64:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP63]], i32 -30 -; POST-PROCESS-NEXT: [[TMP65:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP64]], i32 0, i32 0, i64 43 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP65]], align 4 -; POST-PROCESS-NEXT: [[TMP66:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP67:%.*]] = inttoptr i32 [[TMP66]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP68:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP67]], i32 -30 -; POST-PROCESS-NEXT: [[TMP69:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP68]], i32 0, i32 0, i64 44 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP69]], align 4 -; POST-PROCESS-NEXT: [[TMP70:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP71:%.*]] = inttoptr i32 [[TMP70]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP72:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP71]], i32 -30 -; POST-PROCESS-NEXT: [[TMP73:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP72]], i32 0, i32 0, i64 45 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP73]], align 4 -; POST-PROCESS-NEXT: [[TMP74:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP75:%.*]] = inttoptr i32 [[TMP74]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP76:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP75]], i32 -30 -; POST-PROCESS-NEXT: [[TMP77:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP76]], i32 0, i32 0, i64 46 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP77]], align 4 -; POST-PROCESS-NEXT: [[TMP78:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP79:%.*]] = inttoptr i32 [[TMP78]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP80:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP79]], i32 -30 -; POST-PROCESS-NEXT: [[TMP81:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP80]], i32 0, i32 0, i64 47 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP81]], align 4 -; POST-PROCESS-NEXT: [[TMP82:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP83:%.*]] = inttoptr i32 [[TMP82]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP84:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP83]], i32 -30 -; POST-PROCESS-NEXT: [[TMP85:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP84]], i32 0, i32 0, i64 48 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP85]], align 4 -; POST-PROCESS-NEXT: [[TMP86:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP87:%.*]] = inttoptr i32 [[TMP86]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP88:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP87]], i32 -30 -; POST-PROCESS-NEXT: [[TMP89:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP88]], i32 0, i32 0, i64 49 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP89]], align 4 -; POST-PROCESS-NEXT: [[TMP90:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP91:%.*]] = inttoptr i32 [[TMP90]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP92:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP91]], i32 -30 -; POST-PROCESS-NEXT: [[TMP93:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP92]], i32 0, i32 0, i64 50 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP93]], align 4 -; POST-PROCESS-NEXT: [[TMP94:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP95:%.*]] = inttoptr i32 [[TMP94]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP96:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP95]], i32 -30 -; POST-PROCESS-NEXT: [[TMP97:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP96]], i32 0, i32 0, i64 51 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP97]], align 4 -; POST-PROCESS-NEXT: [[TMP98:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP99:%.*]] = inttoptr i32 [[TMP98]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP100:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP99]], i32 -30 -; POST-PROCESS-NEXT: [[TMP101:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP100]], i32 0, i32 0, i64 52 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP101]], align 4 -; POST-PROCESS-NEXT: [[TMP102:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP103:%.*]] = inttoptr i32 [[TMP102]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP104:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP103]], i32 -30 -; POST-PROCESS-NEXT: [[TMP105:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP104]], i32 0, i32 0, i64 53 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP105]], align 4 -; POST-PROCESS-NEXT: [[TMP106:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP107:%.*]] = inttoptr i32 [[TMP106]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP108:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP107]], i32 -30 -; POST-PROCESS-NEXT: [[TMP109:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP108]], i32 0, i32 0, i64 54 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP109]], align 4 -; POST-PROCESS-NEXT: [[TMP110:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP111:%.*]] = inttoptr i32 [[TMP110]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP112:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP111]], i32 -30 -; POST-PROCESS-NEXT: [[TMP113:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP112]], i32 0, i32 0, i64 55 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP113]], align 4 -; POST-PROCESS-NEXT: [[TMP114:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP115:%.*]] = inttoptr i32 [[TMP114]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP116:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP115]], i32 -30 -; POST-PROCESS-NEXT: [[TMP117:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP116]], i32 0, i32 0, i64 56 -; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP117]], align 4 -; POST-PROCESS-NEXT: [[TMP118:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP119:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @main.resume.0 to i64)) -; POST-PROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 4, i32 [[TMP118]], i64 [[TMP119]], [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]]), !continuation.registercount !17, !continuation.returnedRegistercount !17 +; POST-PROCESS-NEXT: [[TMP12:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP13:%.*]] = inttoptr i32 [[TMP12]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP13]], i32 -30 +; POST-PROCESS-NEXT: [[TMP15:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP14]], i32 0, i32 0, i64 30 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP15]], align 4 +; POST-PROCESS-NEXT: [[TMP16:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP17:%.*]] = inttoptr i32 [[TMP16]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP17]], i32 -30 +; POST-PROCESS-NEXT: [[TMP19:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP18]], i32 0, i32 0, i64 31 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP19]], align 4 +; POST-PROCESS-NEXT: [[TMP20:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP21:%.*]] = inttoptr i32 [[TMP20]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP21]], i32 -30 +; POST-PROCESS-NEXT: [[TMP23:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP22]], i32 0, i32 0, i64 32 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP23]], align 4 +; POST-PROCESS-NEXT: [[TMP24:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP25:%.*]] = inttoptr i32 [[TMP24]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP26:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP25]], i32 -30 +; POST-PROCESS-NEXT: [[TMP27:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP26]], i32 0, i32 0, i64 33 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP27]], align 4 +; POST-PROCESS-NEXT: [[TMP28:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP29:%.*]] = inttoptr i32 [[TMP28]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP30:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP29]], i32 -30 +; POST-PROCESS-NEXT: [[TMP31:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP30]], i32 0, i32 0, i64 34 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP31]], align 4 +; POST-PROCESS-NEXT: [[TMP32:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP33:%.*]] = inttoptr i32 [[TMP32]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP34:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP33]], i32 -30 +; POST-PROCESS-NEXT: [[TMP35:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP34]], i32 0, i32 0, i64 35 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP35]], align 4 +; POST-PROCESS-NEXT: [[TMP36:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP37:%.*]] = inttoptr i32 [[TMP36]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP38:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP37]], i32 -30 +; POST-PROCESS-NEXT: [[TMP39:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP38]], i32 0, i32 0, i64 36 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP39]], align 4 +; POST-PROCESS-NEXT: [[TMP40:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP41:%.*]] = inttoptr i32 [[TMP40]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP42:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP41]], i32 -30 +; POST-PROCESS-NEXT: [[TMP43:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP42]], i32 0, i32 0, i64 37 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP43]], align 4 +; POST-PROCESS-NEXT: [[TMP44:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP45:%.*]] = inttoptr i32 [[TMP44]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP46:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP45]], i32 -30 +; POST-PROCESS-NEXT: [[TMP47:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP46]], i32 0, i32 0, i64 38 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP47]], align 4 +; POST-PROCESS-NEXT: [[TMP48:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP49:%.*]] = inttoptr i32 [[TMP48]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP50:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP49]], i32 -30 +; POST-PROCESS-NEXT: [[TMP51:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP50]], i32 0, i32 0, i64 39 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP51]], align 4 +; POST-PROCESS-NEXT: [[TMP52:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP53:%.*]] = inttoptr i32 [[TMP52]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP54:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP53]], i32 -30 +; POST-PROCESS-NEXT: [[TMP55:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP54]], i32 0, i32 0, i64 40 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP55]], align 4 +; POST-PROCESS-NEXT: [[TMP56:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP57:%.*]] = inttoptr i32 [[TMP56]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP58:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP57]], i32 -30 +; POST-PROCESS-NEXT: [[TMP59:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP58]], i32 0, i32 0, i64 41 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP59]], align 4 +; POST-PROCESS-NEXT: [[TMP60:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP61:%.*]] = inttoptr i32 [[TMP60]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP62:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP61]], i32 -30 +; POST-PROCESS-NEXT: [[TMP63:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP62]], i32 0, i32 0, i64 42 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP63]], align 4 +; POST-PROCESS-NEXT: [[TMP64:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP65:%.*]] = inttoptr i32 [[TMP64]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP66:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP65]], i32 -30 +; POST-PROCESS-NEXT: [[TMP67:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP66]], i32 0, i32 0, i64 43 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP67]], align 4 +; POST-PROCESS-NEXT: [[TMP68:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP69:%.*]] = inttoptr i32 [[TMP68]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP70:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP69]], i32 -30 +; POST-PROCESS-NEXT: [[TMP71:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP70]], i32 0, i32 0, i64 44 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP71]], align 4 +; POST-PROCESS-NEXT: [[TMP72:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP73:%.*]] = inttoptr i32 [[TMP72]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP74:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP73]], i32 -30 +; POST-PROCESS-NEXT: [[TMP75:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP74]], i32 0, i32 0, i64 45 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP75]], align 4 +; POST-PROCESS-NEXT: [[TMP76:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP77:%.*]] = inttoptr i32 [[TMP76]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP78:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP77]], i32 -30 +; POST-PROCESS-NEXT: [[TMP79:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP78]], i32 0, i32 0, i64 46 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP79]], align 4 +; POST-PROCESS-NEXT: [[TMP80:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP81:%.*]] = inttoptr i32 [[TMP80]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP82:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP81]], i32 -30 +; POST-PROCESS-NEXT: [[TMP83:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP82]], i32 0, i32 0, i64 47 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP83]], align 4 +; POST-PROCESS-NEXT: [[TMP84:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP85:%.*]] = inttoptr i32 [[TMP84]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP86:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP85]], i32 -30 +; POST-PROCESS-NEXT: [[TMP87:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP86]], i32 0, i32 0, i64 48 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP87]], align 4 +; POST-PROCESS-NEXT: [[TMP88:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP89:%.*]] = inttoptr i32 [[TMP88]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP90:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP89]], i32 -30 +; POST-PROCESS-NEXT: [[TMP91:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP90]], i32 0, i32 0, i64 49 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP91]], align 4 +; POST-PROCESS-NEXT: [[TMP92:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP93:%.*]] = inttoptr i32 [[TMP92]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP94:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP93]], i32 -30 +; POST-PROCESS-NEXT: [[TMP95:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP94]], i32 0, i32 0, i64 50 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP95]], align 4 +; POST-PROCESS-NEXT: [[TMP96:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP97:%.*]] = inttoptr i32 [[TMP96]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP98:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP97]], i32 -30 +; POST-PROCESS-NEXT: [[TMP99:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP98]], i32 0, i32 0, i64 51 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP99]], align 4 +; POST-PROCESS-NEXT: [[TMP100:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP101:%.*]] = inttoptr i32 [[TMP100]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP102:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP101]], i32 -30 +; POST-PROCESS-NEXT: [[TMP103:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP102]], i32 0, i32 0, i64 52 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP103]], align 4 +; POST-PROCESS-NEXT: [[TMP104:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP105:%.*]] = inttoptr i32 [[TMP104]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP106:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP105]], i32 -30 +; POST-PROCESS-NEXT: [[TMP107:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP106]], i32 0, i32 0, i64 53 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP107]], align 4 +; POST-PROCESS-NEXT: [[TMP108:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP109:%.*]] = inttoptr i32 [[TMP108]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP110:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP109]], i32 -30 +; POST-PROCESS-NEXT: [[TMP111:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP110]], i32 0, i32 0, i64 54 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP111]], align 4 +; POST-PROCESS-NEXT: [[TMP112:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP113:%.*]] = inttoptr i32 [[TMP112]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP114:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP113]], i32 -30 +; POST-PROCESS-NEXT: [[TMP115:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP114]], i32 0, i32 0, i64 55 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP115]], align 4 +; POST-PROCESS-NEXT: [[TMP116:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP117:%.*]] = inttoptr i32 [[TMP116]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP118:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP117]], i32 -30 +; POST-PROCESS-NEXT: [[TMP119:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP118]], i32 0, i32 0, i64 56 +; POST-PROCESS-NEXT: store i32 undef, ptr addrspace(21) [[TMP119]], align 4 +; POST-PROCESS-NEXT: [[TMP120:%.*]] = load i32, ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP121:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @main.resume.0 to i64)) +; POST-PROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 4, i32 [[TMP120]], i64 [[TMP121]], [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]]), !continuation.registercount !17, !continuation.returnedRegistercount !17 ; POST-PROCESS-NEXT: unreachable ; ; -; POST-PROCESS-LABEL: define void @main.resume.0( +; POST-PROCESS-LABEL: define dso_local void @main.resume.0( ; POST-PROCESS-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage !8 !continuation.registercount !17 !continuation !20 { ; POST-PROCESS-NEXT: entryresume.0: ; POST-PROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA]], align 8 @@ -1147,11 +1160,11 @@ attributes #3 = { nounwind } ; POST-PROCESS-NEXT: [[TMP157:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP156]], i32 -30 ; POST-PROCESS-NEXT: [[TMP158:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP157]], i32 0, i32 0, i64 56 ; POST-PROCESS-NEXT: [[TMP159:%.*]] = load i32, ptr addrspace(21) [[TMP158]], align 4 +; POST-PROCESS-NEXT: [[DOTFCA_0_EXTRACT1:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], 0 +; POST-PROCESS-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; POST-PROCESS-NEXT: [[TMP160:%.*]] = load i32, ptr [[CSP]], align 4 ; POST-PROCESS-NEXT: [[TMP161:%.*]] = add i32 [[TMP160]], -108 ; POST-PROCESS-NEXT: store i32 [[TMP161]], ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[DOTFCA_0_EXTRACT1:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], 0 -; POST-PROCESS-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; POST-PROCESS-NEXT: call void @continuation.complete() ; POST-PROCESS-NEXT: unreachable ; @@ -1535,400 +1548,384 @@ attributes #3 = { nounwind } ; POST-PROCESS-NEXT: [[CSP:%.*]] = alloca i32, align 4 ; POST-PROCESS-NEXT: store [[STRUCT_SYSTEMDATA]] [[TMP0]], ptr [[SYSTEM_DATA]], align 4 ; POST-PROCESS-NEXT: store i32 [[CSPINIT]], ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP1:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 0 -; POST-PROCESS-NEXT: store i32 [[TMP2]], ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP1]] to ptr addrspace(21) ; POST-PROCESS-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = getelementptr inbounds [[CLOSESTHIT_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; POST-PROCESS-NEXT: [[RETURNADDR_SPILL_ADDR:%.*]] = getelementptr inbounds [[CLOSESTHIT_FRAME]], ptr [[CONT_STATE]], i32 0, i32 1 ; POST-PROCESS-NEXT: store i64 [[RETURNADDR]], ptr [[RETURNADDR_SPILL_ADDR]], align 4 -; POST-PROCESS-NEXT: [[TMP4:%.*]] = load [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA]], align 4 -; POST-PROCESS-NEXT: [[DOTFCA_0_0_EXTRACT:%.*]] = extractvalue [[STRUCT_SYSTEMDATA]] [[TMP4]], 0, 0 +; POST-PROCESS-NEXT: [[TMP1:%.*]] = load [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA]], align 4 +; POST-PROCESS-NEXT: [[DOTFCA_0_0_EXTRACT:%.*]] = extractvalue [[STRUCT_SYSTEMDATA]] [[TMP1]], 0, 0 ; POST-PROCESS-NEXT: [[DOTFCA_0_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 ; POST-PROCESS-NEXT: store i32 [[DOTFCA_0_0_EXTRACT]], ptr [[DOTFCA_0_0_GEP]], align 4 -; POST-PROCESS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; POST-PROCESS-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 ; POST-PROCESS-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) -; POST-PROCESS-NEXT: [[TMP6:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S:%.*]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i32 7) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 8) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP8:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 9) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP9:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 10) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP10:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 11) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP11:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 12) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP12:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 13) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP13:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 14) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP14:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 15) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP15:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 16) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP16:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 17) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP17:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 18) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP18:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 19) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP19:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 20) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP20:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 21) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP21:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 22) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP22:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 23) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP23:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 24) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP24:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 25) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP25:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 26) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP26:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 27) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP27:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 28) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP28:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 29) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP29:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP30:%.*]] = inttoptr i32 [[TMP29]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP31:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP30]], i32 -30 -; POST-PROCESS-NEXT: [[TMP32:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP31]], i32 0, i32 0, i64 30 -; POST-PROCESS-NEXT: [[TMP33:%.*]] = load i32, ptr addrspace(21) [[TMP32]], align 4 -; POST-PROCESS-NEXT: [[TMP34:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP35:%.*]] = inttoptr i32 [[TMP34]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP36:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP35]], i32 -30 -; POST-PROCESS-NEXT: [[TMP37:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP36]], i32 0, i32 0, i64 31 -; POST-PROCESS-NEXT: [[TMP38:%.*]] = load i32, ptr addrspace(21) [[TMP37]], align 4 -; POST-PROCESS-NEXT: [[TMP39:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP40:%.*]] = inttoptr i32 [[TMP39]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP41:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP40]], i32 -30 -; POST-PROCESS-NEXT: [[TMP42:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP41]], i32 0, i32 0, i64 32 -; POST-PROCESS-NEXT: [[TMP43:%.*]] = load i32, ptr addrspace(21) [[TMP42]], align 4 -; POST-PROCESS-NEXT: [[TMP44:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP45:%.*]] = inttoptr i32 [[TMP44]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP46:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP45]], i32 -30 -; POST-PROCESS-NEXT: [[TMP47:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP46]], i32 0, i32 0, i64 33 -; POST-PROCESS-NEXT: [[TMP48:%.*]] = load i32, ptr addrspace(21) [[TMP47]], align 4 -; POST-PROCESS-NEXT: [[TMP49:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP50:%.*]] = inttoptr i32 [[TMP49]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP51:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP50]], i32 -30 -; POST-PROCESS-NEXT: [[TMP52:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP51]], i32 0, i32 0, i64 34 -; POST-PROCESS-NEXT: [[TMP53:%.*]] = load i32, ptr addrspace(21) [[TMP52]], align 4 -; POST-PROCESS-NEXT: [[TMP54:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP55:%.*]] = inttoptr i32 [[TMP54]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP56:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP55]], i32 -30 -; POST-PROCESS-NEXT: [[TMP57:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP56]], i32 0, i32 0, i64 35 -; POST-PROCESS-NEXT: [[TMP58:%.*]] = load i32, ptr addrspace(21) [[TMP57]], align 4 -; POST-PROCESS-NEXT: [[TMP59:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP60:%.*]] = inttoptr i32 [[TMP59]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP61:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP60]], i32 -30 -; POST-PROCESS-NEXT: [[TMP62:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP61]], i32 0, i32 0, i64 36 -; POST-PROCESS-NEXT: [[TMP63:%.*]] = load i32, ptr addrspace(21) [[TMP62]], align 4 -; POST-PROCESS-NEXT: [[TMP64:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP65:%.*]] = inttoptr i32 [[TMP64]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP66:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP65]], i32 -30 -; POST-PROCESS-NEXT: [[TMP67:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP66]], i32 0, i32 0, i64 37 -; POST-PROCESS-NEXT: [[TMP68:%.*]] = load i32, ptr addrspace(21) [[TMP67]], align 4 -; POST-PROCESS-NEXT: [[TMP69:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP70:%.*]] = inttoptr i32 [[TMP69]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP71:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP70]], i32 -30 -; POST-PROCESS-NEXT: [[TMP72:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP71]], i32 0, i32 0, i64 38 -; POST-PROCESS-NEXT: [[TMP73:%.*]] = load i32, ptr addrspace(21) [[TMP72]], align 4 -; POST-PROCESS-NEXT: [[TMP74:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP75:%.*]] = inttoptr i32 [[TMP74]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP76:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP75]], i32 -30 -; POST-PROCESS-NEXT: [[TMP77:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP76]], i32 0, i32 0, i64 39 -; POST-PROCESS-NEXT: [[TMP78:%.*]] = load i32, ptr addrspace(21) [[TMP77]], align 4 -; POST-PROCESS-NEXT: [[TMP79:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP80:%.*]] = inttoptr i32 [[TMP79]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP81:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP80]], i32 -30 -; POST-PROCESS-NEXT: [[TMP82:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP81]], i32 0, i32 0, i64 40 -; POST-PROCESS-NEXT: [[TMP83:%.*]] = load i32, ptr addrspace(21) [[TMP82]], align 4 -; POST-PROCESS-NEXT: [[TMP84:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP85:%.*]] = inttoptr i32 [[TMP84]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP86:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP85]], i32 -30 -; POST-PROCESS-NEXT: [[TMP87:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP86]], i32 0, i32 0, i64 41 -; POST-PROCESS-NEXT: [[TMP88:%.*]] = load i32, ptr addrspace(21) [[TMP87]], align 4 -; POST-PROCESS-NEXT: [[TMP89:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP90:%.*]] = inttoptr i32 [[TMP89]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP91:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP90]], i32 -30 -; POST-PROCESS-NEXT: [[TMP92:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP91]], i32 0, i32 0, i64 42 -; POST-PROCESS-NEXT: [[TMP93:%.*]] = load i32, ptr addrspace(21) [[TMP92]], align 4 -; POST-PROCESS-NEXT: [[TMP94:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP95:%.*]] = inttoptr i32 [[TMP94]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP96:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP95]], i32 -30 -; POST-PROCESS-NEXT: [[TMP97:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP96]], i32 0, i32 0, i64 43 -; POST-PROCESS-NEXT: [[TMP98:%.*]] = load i32, ptr addrspace(21) [[TMP97]], align 4 -; POST-PROCESS-NEXT: [[TMP99:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP100:%.*]] = inttoptr i32 [[TMP99]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP101:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP100]], i32 -30 -; POST-PROCESS-NEXT: [[TMP102:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP101]], i32 0, i32 0, i64 44 -; POST-PROCESS-NEXT: [[TMP103:%.*]] = load i32, ptr addrspace(21) [[TMP102]], align 4 -; POST-PROCESS-NEXT: [[TMP104:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP105:%.*]] = inttoptr i32 [[TMP104]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP106:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP105]], i32 -30 -; POST-PROCESS-NEXT: [[TMP107:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP106]], i32 0, i32 0, i64 45 -; POST-PROCESS-NEXT: [[TMP108:%.*]] = load i32, ptr addrspace(21) [[TMP107]], align 4 -; POST-PROCESS-NEXT: [[TMP109:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP110:%.*]] = inttoptr i32 [[TMP109]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP111:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP110]], i32 -30 -; POST-PROCESS-NEXT: [[TMP112:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP111]], i32 0, i32 0, i64 46 -; POST-PROCESS-NEXT: [[TMP113:%.*]] = load i32, ptr addrspace(21) [[TMP112]], align 4 -; POST-PROCESS-NEXT: [[TMP114:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP115:%.*]] = inttoptr i32 [[TMP114]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP116:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP115]], i32 -30 -; POST-PROCESS-NEXT: [[TMP117:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP116]], i32 0, i32 0, i64 47 -; POST-PROCESS-NEXT: [[TMP118:%.*]] = load i32, ptr addrspace(21) [[TMP117]], align 4 -; POST-PROCESS-NEXT: [[TMP119:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP120:%.*]] = inttoptr i32 [[TMP119]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP121:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP120]], i32 -30 -; POST-PROCESS-NEXT: [[TMP122:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP121]], i32 0, i32 0, i64 48 -; POST-PROCESS-NEXT: [[TMP123:%.*]] = load i32, ptr addrspace(21) [[TMP122]], align 4 -; POST-PROCESS-NEXT: [[TMP124:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP125:%.*]] = inttoptr i32 [[TMP124]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP126:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP125]], i32 -30 -; POST-PROCESS-NEXT: [[TMP127:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP126]], i32 0, i32 0, i64 49 -; POST-PROCESS-NEXT: [[TMP128:%.*]] = load i32, ptr addrspace(21) [[TMP127]], align 4 -; POST-PROCESS-NEXT: [[TMP129:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP130:%.*]] = inttoptr i32 [[TMP129]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP131:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP130]], i32 -30 -; POST-PROCESS-NEXT: [[TMP132:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP131]], i32 0, i32 0, i64 50 -; POST-PROCESS-NEXT: [[TMP133:%.*]] = load i32, ptr addrspace(21) [[TMP132]], align 4 -; POST-PROCESS-NEXT: [[TMP134:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP135:%.*]] = inttoptr i32 [[TMP134]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP136:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP135]], i32 -30 -; POST-PROCESS-NEXT: [[TMP137:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP136]], i32 0, i32 0, i64 51 -; POST-PROCESS-NEXT: [[TMP138:%.*]] = load i32, ptr addrspace(21) [[TMP137]], align 4 -; POST-PROCESS-NEXT: [[TMP139:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP140:%.*]] = inttoptr i32 [[TMP139]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP141:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP140]], i32 -30 -; POST-PROCESS-NEXT: [[TMP142:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP141]], i32 0, i32 0, i64 52 -; POST-PROCESS-NEXT: [[TMP143:%.*]] = load i32, ptr addrspace(21) [[TMP142]], align 4 -; POST-PROCESS-NEXT: [[TMP144:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP145:%.*]] = inttoptr i32 [[TMP144]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP146:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP145]], i32 -30 -; POST-PROCESS-NEXT: [[TMP147:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP146]], i32 0, i32 0, i64 53 -; POST-PROCESS-NEXT: [[TMP148:%.*]] = load i32, ptr addrspace(21) [[TMP147]], align 4 -; POST-PROCESS-NEXT: [[TMP149:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP150:%.*]] = inttoptr i32 [[TMP149]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP151:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP150]], i32 -30 -; POST-PROCESS-NEXT: [[TMP152:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP151]], i32 0, i32 0, i64 54 -; POST-PROCESS-NEXT: [[TMP153:%.*]] = load i32, ptr addrspace(21) [[TMP152]], align 4 -; POST-PROCESS-NEXT: [[TMP154:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP155:%.*]] = inttoptr i32 [[TMP154]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP156:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP155]], i32 -30 -; POST-PROCESS-NEXT: [[TMP157:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP156]], i32 0, i32 0, i64 55 -; POST-PROCESS-NEXT: [[TMP158:%.*]] = load i32, ptr addrspace(21) [[TMP157]], align 4 -; POST-PROCESS-NEXT: [[TMP159:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP160:%.*]] = inttoptr i32 [[TMP159]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP161:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP160]], i32 -30 -; POST-PROCESS-NEXT: [[TMP162:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP161]], i32 0, i32 0, i64 56 -; POST-PROCESS-NEXT: [[TMP163:%.*]] = load i32, ptr addrspace(21) [[TMP162]], align 4 -; POST-PROCESS-NEXT: [[TMP164:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP3:%.*]] = load i32, ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 108 +; POST-PROCESS-NEXT: store i32 [[TMP4]], ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP5:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S:%.*]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i32 7) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP6:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 8) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 9) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP8:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 10) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP9:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 11) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP10:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 12) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP11:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 13) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP12:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 14) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP13:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 15) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP14:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 16) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP15:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 17) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP16:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 18) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP17:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 19) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP18:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 20) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP19:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 21) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP20:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 22) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP21:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 23) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP22:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 24) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP23:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 25) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP24:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 26) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP25:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 27) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP26:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 28) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP27:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 29) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP28:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP29:%.*]] = inttoptr i32 [[TMP28]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP30:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP29]], i32 -30 +; POST-PROCESS-NEXT: [[TMP31:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP30]], i32 0, i32 0, i64 30 +; POST-PROCESS-NEXT: [[TMP32:%.*]] = load i32, ptr addrspace(21) [[TMP31]], align 4 +; POST-PROCESS-NEXT: [[TMP33:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP34:%.*]] = inttoptr i32 [[TMP33]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP35:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP34]], i32 -30 +; POST-PROCESS-NEXT: [[TMP36:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP35]], i32 0, i32 0, i64 31 +; POST-PROCESS-NEXT: [[TMP37:%.*]] = load i32, ptr addrspace(21) [[TMP36]], align 4 +; POST-PROCESS-NEXT: [[TMP38:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP39:%.*]] = inttoptr i32 [[TMP38]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP40:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP39]], i32 -30 +; POST-PROCESS-NEXT: [[TMP41:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP40]], i32 0, i32 0, i64 32 +; POST-PROCESS-NEXT: [[TMP42:%.*]] = load i32, ptr addrspace(21) [[TMP41]], align 4 +; POST-PROCESS-NEXT: [[TMP43:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP44:%.*]] = inttoptr i32 [[TMP43]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP45:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP44]], i32 -30 +; POST-PROCESS-NEXT: [[TMP46:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP45]], i32 0, i32 0, i64 33 +; POST-PROCESS-NEXT: [[TMP47:%.*]] = load i32, ptr addrspace(21) [[TMP46]], align 4 +; POST-PROCESS-NEXT: [[TMP48:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP49:%.*]] = inttoptr i32 [[TMP48]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP50:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP49]], i32 -30 +; POST-PROCESS-NEXT: [[TMP51:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP50]], i32 0, i32 0, i64 34 +; POST-PROCESS-NEXT: [[TMP52:%.*]] = load i32, ptr addrspace(21) [[TMP51]], align 4 +; POST-PROCESS-NEXT: [[TMP53:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP54:%.*]] = inttoptr i32 [[TMP53]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP55:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP54]], i32 -30 +; POST-PROCESS-NEXT: [[TMP56:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP55]], i32 0, i32 0, i64 35 +; POST-PROCESS-NEXT: [[TMP57:%.*]] = load i32, ptr addrspace(21) [[TMP56]], align 4 +; POST-PROCESS-NEXT: [[TMP58:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP59:%.*]] = inttoptr i32 [[TMP58]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP60:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP59]], i32 -30 +; POST-PROCESS-NEXT: [[TMP61:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP60]], i32 0, i32 0, i64 36 +; POST-PROCESS-NEXT: [[TMP62:%.*]] = load i32, ptr addrspace(21) [[TMP61]], align 4 +; POST-PROCESS-NEXT: [[TMP63:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP64:%.*]] = inttoptr i32 [[TMP63]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP65:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP64]], i32 -30 +; POST-PROCESS-NEXT: [[TMP66:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP65]], i32 0, i32 0, i64 37 +; POST-PROCESS-NEXT: [[TMP67:%.*]] = load i32, ptr addrspace(21) [[TMP66]], align 4 +; POST-PROCESS-NEXT: [[TMP68:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP69:%.*]] = inttoptr i32 [[TMP68]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP70:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP69]], i32 -30 +; POST-PROCESS-NEXT: [[TMP71:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP70]], i32 0, i32 0, i64 38 +; POST-PROCESS-NEXT: [[TMP72:%.*]] = load i32, ptr addrspace(21) [[TMP71]], align 4 +; POST-PROCESS-NEXT: [[TMP73:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP74:%.*]] = inttoptr i32 [[TMP73]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP75:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP74]], i32 -30 +; POST-PROCESS-NEXT: [[TMP76:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP75]], i32 0, i32 0, i64 39 +; POST-PROCESS-NEXT: [[TMP77:%.*]] = load i32, ptr addrspace(21) [[TMP76]], align 4 +; POST-PROCESS-NEXT: [[TMP78:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP79:%.*]] = inttoptr i32 [[TMP78]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP80:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP79]], i32 -30 +; POST-PROCESS-NEXT: [[TMP81:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP80]], i32 0, i32 0, i64 40 +; POST-PROCESS-NEXT: [[TMP82:%.*]] = load i32, ptr addrspace(21) [[TMP81]], align 4 +; POST-PROCESS-NEXT: [[TMP83:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP84:%.*]] = inttoptr i32 [[TMP83]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP85:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP84]], i32 -30 +; POST-PROCESS-NEXT: [[TMP86:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP85]], i32 0, i32 0, i64 41 +; POST-PROCESS-NEXT: [[TMP87:%.*]] = load i32, ptr addrspace(21) [[TMP86]], align 4 +; POST-PROCESS-NEXT: [[TMP88:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP89:%.*]] = inttoptr i32 [[TMP88]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP90:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP89]], i32 -30 +; POST-PROCESS-NEXT: [[TMP91:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP90]], i32 0, i32 0, i64 42 +; POST-PROCESS-NEXT: [[TMP92:%.*]] = load i32, ptr addrspace(21) [[TMP91]], align 4 +; POST-PROCESS-NEXT: [[TMP93:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP94:%.*]] = inttoptr i32 [[TMP93]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP95:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP94]], i32 -30 +; POST-PROCESS-NEXT: [[TMP96:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP95]], i32 0, i32 0, i64 43 +; POST-PROCESS-NEXT: [[TMP97:%.*]] = load i32, ptr addrspace(21) [[TMP96]], align 4 +; POST-PROCESS-NEXT: [[TMP98:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP99:%.*]] = inttoptr i32 [[TMP98]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP100:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP99]], i32 -30 +; POST-PROCESS-NEXT: [[TMP101:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP100]], i32 0, i32 0, i64 44 +; POST-PROCESS-NEXT: [[TMP102:%.*]] = load i32, ptr addrspace(21) [[TMP101]], align 4 +; POST-PROCESS-NEXT: [[TMP103:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP104:%.*]] = inttoptr i32 [[TMP103]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP105:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP104]], i32 -30 +; POST-PROCESS-NEXT: [[TMP106:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP105]], i32 0, i32 0, i64 45 +; POST-PROCESS-NEXT: [[TMP107:%.*]] = load i32, ptr addrspace(21) [[TMP106]], align 4 +; POST-PROCESS-NEXT: [[TMP108:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP109:%.*]] = inttoptr i32 [[TMP108]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP110:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP109]], i32 -30 +; POST-PROCESS-NEXT: [[TMP111:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP110]], i32 0, i32 0, i64 46 +; POST-PROCESS-NEXT: [[TMP112:%.*]] = load i32, ptr addrspace(21) [[TMP111]], align 4 +; POST-PROCESS-NEXT: [[TMP113:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP114:%.*]] = inttoptr i32 [[TMP113]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP115:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP114]], i32 -30 +; POST-PROCESS-NEXT: [[TMP116:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP115]], i32 0, i32 0, i64 47 +; POST-PROCESS-NEXT: [[TMP117:%.*]] = load i32, ptr addrspace(21) [[TMP116]], align 4 +; POST-PROCESS-NEXT: [[TMP118:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP119:%.*]] = inttoptr i32 [[TMP118]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP120:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP119]], i32 -30 +; POST-PROCESS-NEXT: [[TMP121:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP120]], i32 0, i32 0, i64 48 +; POST-PROCESS-NEXT: [[TMP122:%.*]] = load i32, ptr addrspace(21) [[TMP121]], align 4 +; POST-PROCESS-NEXT: [[TMP123:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP124:%.*]] = inttoptr i32 [[TMP123]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP125:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP124]], i32 -30 +; POST-PROCESS-NEXT: [[TMP126:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP125]], i32 0, i32 0, i64 49 +; POST-PROCESS-NEXT: [[TMP127:%.*]] = load i32, ptr addrspace(21) [[TMP126]], align 4 +; POST-PROCESS-NEXT: [[TMP128:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP129:%.*]] = inttoptr i32 [[TMP128]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP130:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP129]], i32 -30 +; POST-PROCESS-NEXT: [[TMP131:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP130]], i32 0, i32 0, i64 50 +; POST-PROCESS-NEXT: [[TMP132:%.*]] = load i32, ptr addrspace(21) [[TMP131]], align 4 +; POST-PROCESS-NEXT: [[TMP133:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP134:%.*]] = inttoptr i32 [[TMP133]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP135:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP134]], i32 -30 +; POST-PROCESS-NEXT: [[TMP136:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP135]], i32 0, i32 0, i64 51 +; POST-PROCESS-NEXT: [[TMP137:%.*]] = load i32, ptr addrspace(21) [[TMP136]], align 4 +; POST-PROCESS-NEXT: [[TMP138:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP139:%.*]] = inttoptr i32 [[TMP138]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP140:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP139]], i32 -30 +; POST-PROCESS-NEXT: [[TMP141:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP140]], i32 0, i32 0, i64 52 +; POST-PROCESS-NEXT: [[TMP142:%.*]] = load i32, ptr addrspace(21) [[TMP141]], align 4 +; POST-PROCESS-NEXT: [[TMP143:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP144:%.*]] = inttoptr i32 [[TMP143]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP145:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP144]], i32 -30 +; POST-PROCESS-NEXT: [[TMP146:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP145]], i32 0, i32 0, i64 53 +; POST-PROCESS-NEXT: [[TMP147:%.*]] = load i32, ptr addrspace(21) [[TMP146]], align 4 +; POST-PROCESS-NEXT: [[TMP148:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP149:%.*]] = inttoptr i32 [[TMP148]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP150:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP149]], i32 -30 +; POST-PROCESS-NEXT: [[TMP151:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP150]], i32 0, i32 0, i64 54 +; POST-PROCESS-NEXT: [[TMP152:%.*]] = load i32, ptr addrspace(21) [[TMP151]], align 4 +; POST-PROCESS-NEXT: [[TMP153:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP154:%.*]] = inttoptr i32 [[TMP153]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP155:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP154]], i32 -30 +; POST-PROCESS-NEXT: [[TMP156:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP155]], i32 0, i32 0, i64 55 +; POST-PROCESS-NEXT: [[TMP157:%.*]] = load i32, ptr addrspace(21) [[TMP156]], align 4 +; POST-PROCESS-NEXT: [[TMP158:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP159:%.*]] = inttoptr i32 [[TMP158]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP160:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP159]], i32 -30 +; POST-PROCESS-NEXT: [[TMP161:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr addrspace(21) [[TMP160]], i32 0, i32 0, i64 56 +; POST-PROCESS-NEXT: [[TMP162:%.*]] = load i32, ptr addrspace(21) [[TMP161]], align 4 +; POST-PROCESS-NEXT: [[TMP163:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 ; POST-PROCESS-NEXT: [[DOTSPILL_ADDR:%.*]] = getelementptr inbounds [[CLOSESTHIT_FRAME]], ptr [[CONT_STATE]], i32 0, i32 2 -; POST-PROCESS-NEXT: store i32 [[TMP164]], ptr [[DOTSPILL_ADDR]], align 4 -; POST-PROCESS-NEXT: [[TMP165:%.*]] = call [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] @_cont_GetTriangleHitAttributes(ptr [[SYSTEM_DATA_ALLOCA]]) -; POST-PROCESS-NEXT: [[DOTFCA_0_EXTRACT52:%.*]] = extractvalue [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP165]], 0 +; POST-PROCESS-NEXT: store i32 [[TMP163]], ptr [[DOTSPILL_ADDR]], align 4 +; POST-PROCESS-NEXT: [[TMP164:%.*]] = call [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] @_cont_GetTriangleHitAttributes(ptr [[SYSTEM_DATA_ALLOCA]]) +; POST-PROCESS-NEXT: [[DOTFCA_0_EXTRACT52:%.*]] = extractvalue [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP164]], 0 ; POST-PROCESS-NEXT: [[DOTSROA_054_0_VEC_EXTRACT:%.*]] = extractelement <2 x float> [[DOTFCA_0_EXTRACT52]], i32 0 -; POST-PROCESS-NEXT: [[TMP166:%.*]] = bitcast float [[DOTSROA_054_0_VEC_EXTRACT]] to i32 +; POST-PROCESS-NEXT: [[TMP165:%.*]] = bitcast float [[DOTSROA_054_0_VEC_EXTRACT]] to i32 ; POST-PROCESS-NEXT: [[DOTSROA_054_4_VEC_EXTRACT:%.*]] = extractelement <2 x float> [[DOTFCA_0_EXTRACT52]], i32 1 -; POST-PROCESS-NEXT: [[TMP167:%.*]] = bitcast float [[DOTSROA_054_4_VEC_EXTRACT]] to i32 -; POST-PROCESS-NEXT: [[TMP168:%.*]] = load [[DX_TYPES_HANDLE:%.*]], ptr @"\01?Scene@@3URaytracingAccelerationStructure@@A", align 4 -; POST-PROCESS-NEXT: [[TMP169:%.*]] = load [[DX_TYPES_HANDLE]], ptr @"\01?RenderTarget@@3V?$RWTexture2D@V?$vector@M$03@@@@A", align 4 -; POST-PROCESS-NEXT: [[TMP170:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP168]]) -; POST-PROCESS-NEXT: [[TMP171:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP170]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) -; POST-PROCESS-NEXT: [[TMP172:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[TMP171]]) -; POST-PROCESS-NEXT: [[TMP173:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; POST-PROCESS-NEXT: [[DIS_DATA_I_FCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP173]], i32 0, i32 0 +; POST-PROCESS-NEXT: [[TMP166:%.*]] = bitcast float [[DOTSROA_054_4_VEC_EXTRACT]] to i32 +; POST-PROCESS-NEXT: [[TMP167:%.*]] = load [[DX_TYPES_HANDLE:%.*]], ptr @"\01?Scene@@3URaytracingAccelerationStructure@@A", align 4 +; POST-PROCESS-NEXT: [[TMP168:%.*]] = load [[DX_TYPES_HANDLE]], ptr @"\01?RenderTarget@@3V?$RWTexture2D@V?$vector@M$03@@@@A", align 4 +; POST-PROCESS-NEXT: [[TMP169:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP167]]) +; POST-PROCESS-NEXT: [[TMP170:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP169]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) +; POST-PROCESS-NEXT: [[TMP171:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[TMP170]]) +; POST-PROCESS-NEXT: [[TMP172:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; POST-PROCESS-NEXT: [[DIS_DATA_I_FCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP172]], i32 0, i32 0 ; POST-PROCESS-NEXT: [[DIS_DATA_I_FCA_0_LOAD:%.*]] = load i32, ptr [[DIS_DATA_I_FCA_0_GEP]], align 4 ; POST-PROCESS-NEXT: [[DIS_DATA_I_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, i32 [[DIS_DATA_I_FCA_0_LOAD]], 0 ; POST-PROCESS-NEXT: [[SYS_DATA_I:%.*]] = insertvalue [[STRUCT_SYSTEMDATA]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I_FCA_0_INSERT]], 0 ; POST-PROCESS-NEXT: [[TRAV_DATA_I:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA_I]], 0 -; POST-PROCESS-NEXT: [[TMP174:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP175:%.*]] = add i32 [[TMP174]], 108 -; POST-PROCESS-NEXT: store i32 [[TMP175]], ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP173:%.*]] = load i32, ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP174:%.*]] = add i32 [[TMP173]], -108 ; POST-PROCESS-NEXT: store i32 [[TMP174]], ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: store i32 [[TMP6]], ptr addrspace(20) addrspacecast (ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT:%.*]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i32 7) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP7]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 8) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP8]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 9) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP9]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 10) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP10]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 11) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP11]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 12) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP12]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 13) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP13]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 14) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP14]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 15) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP15]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 16) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP16]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 17) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP17]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 18) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP18]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 19) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP19]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 20) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP20]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 21) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP21]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 22) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP22]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 23) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP23]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 24) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP24]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 25) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP25]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 26) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP26]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 27) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP27]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 28) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP28]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 29) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP176:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP177:%.*]] = inttoptr i32 [[TMP176]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP178:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP177]], i32 -30 -; POST-PROCESS-NEXT: [[TMP179:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP178]], i32 0, i32 0, i64 30 -; POST-PROCESS-NEXT: store i32 [[TMP33]], ptr addrspace(21) [[TMP179]], align 4 -; POST-PROCESS-NEXT: [[TMP180:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP181:%.*]] = inttoptr i32 [[TMP180]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP182:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP181]], i32 -30 -; POST-PROCESS-NEXT: [[TMP183:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP182]], i32 0, i32 0, i64 31 -; POST-PROCESS-NEXT: store i32 [[TMP38]], ptr addrspace(21) [[TMP183]], align 4 -; POST-PROCESS-NEXT: [[TMP184:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP185:%.*]] = inttoptr i32 [[TMP184]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP186:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP185]], i32 -30 -; POST-PROCESS-NEXT: [[TMP187:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP186]], i32 0, i32 0, i64 32 -; POST-PROCESS-NEXT: store i32 [[TMP43]], ptr addrspace(21) [[TMP187]], align 4 -; POST-PROCESS-NEXT: [[TMP188:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP189:%.*]] = inttoptr i32 [[TMP188]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP190:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP189]], i32 -30 -; POST-PROCESS-NEXT: [[TMP191:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP190]], i32 0, i32 0, i64 33 -; POST-PROCESS-NEXT: store i32 [[TMP48]], ptr addrspace(21) [[TMP191]], align 4 -; POST-PROCESS-NEXT: [[TMP192:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP193:%.*]] = inttoptr i32 [[TMP192]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP194:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP193]], i32 -30 -; POST-PROCESS-NEXT: [[TMP195:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP194]], i32 0, i32 0, i64 34 -; POST-PROCESS-NEXT: store i32 [[TMP53]], ptr addrspace(21) [[TMP195]], align 4 -; POST-PROCESS-NEXT: [[TMP196:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP197:%.*]] = inttoptr i32 [[TMP196]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP198:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP197]], i32 -30 -; POST-PROCESS-NEXT: [[TMP199:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP198]], i32 0, i32 0, i64 35 -; POST-PROCESS-NEXT: store i32 [[TMP58]], ptr addrspace(21) [[TMP199]], align 4 -; POST-PROCESS-NEXT: [[TMP200:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP201:%.*]] = inttoptr i32 [[TMP200]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP202:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP201]], i32 -30 -; POST-PROCESS-NEXT: [[TMP203:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP202]], i32 0, i32 0, i64 36 -; POST-PROCESS-NEXT: store i32 [[TMP63]], ptr addrspace(21) [[TMP203]], align 4 -; POST-PROCESS-NEXT: [[TMP204:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP205:%.*]] = inttoptr i32 [[TMP204]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP206:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP205]], i32 -30 -; POST-PROCESS-NEXT: [[TMP207:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP206]], i32 0, i32 0, i64 37 -; POST-PROCESS-NEXT: store i32 [[TMP68]], ptr addrspace(21) [[TMP207]], align 4 -; POST-PROCESS-NEXT: [[TMP208:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP209:%.*]] = inttoptr i32 [[TMP208]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP210:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP209]], i32 -30 -; POST-PROCESS-NEXT: [[TMP211:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP210]], i32 0, i32 0, i64 38 -; POST-PROCESS-NEXT: store i32 [[TMP73]], ptr addrspace(21) [[TMP211]], align 4 -; POST-PROCESS-NEXT: [[TMP212:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP213:%.*]] = inttoptr i32 [[TMP212]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP214:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP213]], i32 -30 -; POST-PROCESS-NEXT: [[TMP215:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP214]], i32 0, i32 0, i64 39 -; POST-PROCESS-NEXT: store i32 [[TMP78]], ptr addrspace(21) [[TMP215]], align 4 -; POST-PROCESS-NEXT: [[TMP216:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP217:%.*]] = inttoptr i32 [[TMP216]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP218:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP217]], i32 -30 -; POST-PROCESS-NEXT: [[TMP219:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP218]], i32 0, i32 0, i64 40 -; POST-PROCESS-NEXT: store i32 [[TMP83]], ptr addrspace(21) [[TMP219]], align 4 -; POST-PROCESS-NEXT: [[TMP220:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP221:%.*]] = inttoptr i32 [[TMP220]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP222:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP221]], i32 -30 -; POST-PROCESS-NEXT: [[TMP223:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP222]], i32 0, i32 0, i64 41 -; POST-PROCESS-NEXT: store i32 [[TMP88]], ptr addrspace(21) [[TMP223]], align 4 -; POST-PROCESS-NEXT: [[TMP224:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP225:%.*]] = inttoptr i32 [[TMP224]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP226:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP225]], i32 -30 -; POST-PROCESS-NEXT: [[TMP227:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP226]], i32 0, i32 0, i64 42 -; POST-PROCESS-NEXT: store i32 [[TMP93]], ptr addrspace(21) [[TMP227]], align 4 -; POST-PROCESS-NEXT: [[TMP228:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP229:%.*]] = inttoptr i32 [[TMP228]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP230:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP229]], i32 -30 -; POST-PROCESS-NEXT: [[TMP231:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP230]], i32 0, i32 0, i64 43 -; POST-PROCESS-NEXT: store i32 [[TMP98]], ptr addrspace(21) [[TMP231]], align 4 -; POST-PROCESS-NEXT: [[TMP232:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP233:%.*]] = inttoptr i32 [[TMP232]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP234:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP233]], i32 -30 -; POST-PROCESS-NEXT: [[TMP235:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP234]], i32 0, i32 0, i64 44 -; POST-PROCESS-NEXT: store i32 [[TMP103]], ptr addrspace(21) [[TMP235]], align 4 -; POST-PROCESS-NEXT: [[TMP236:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP237:%.*]] = inttoptr i32 [[TMP236]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP238:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP237]], i32 -30 -; POST-PROCESS-NEXT: [[TMP239:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP238]], i32 0, i32 0, i64 45 -; POST-PROCESS-NEXT: store i32 [[TMP108]], ptr addrspace(21) [[TMP239]], align 4 -; POST-PROCESS-NEXT: [[TMP240:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP241:%.*]] = inttoptr i32 [[TMP240]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP242:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP241]], i32 -30 -; POST-PROCESS-NEXT: [[TMP243:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP242]], i32 0, i32 0, i64 46 -; POST-PROCESS-NEXT: store i32 [[TMP113]], ptr addrspace(21) [[TMP243]], align 4 -; POST-PROCESS-NEXT: [[TMP244:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP245:%.*]] = inttoptr i32 [[TMP244]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP246:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP245]], i32 -30 -; POST-PROCESS-NEXT: [[TMP247:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP246]], i32 0, i32 0, i64 47 -; POST-PROCESS-NEXT: store i32 [[TMP118]], ptr addrspace(21) [[TMP247]], align 4 -; POST-PROCESS-NEXT: [[TMP248:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP249:%.*]] = inttoptr i32 [[TMP248]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP250:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP249]], i32 -30 -; POST-PROCESS-NEXT: [[TMP251:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP250]], i32 0, i32 0, i64 48 -; POST-PROCESS-NEXT: store i32 [[TMP123]], ptr addrspace(21) [[TMP251]], align 4 -; POST-PROCESS-NEXT: [[TMP252:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP253:%.*]] = inttoptr i32 [[TMP252]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP254:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP253]], i32 -30 -; POST-PROCESS-NEXT: [[TMP255:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP254]], i32 0, i32 0, i64 49 -; POST-PROCESS-NEXT: store i32 [[TMP128]], ptr addrspace(21) [[TMP255]], align 4 -; POST-PROCESS-NEXT: [[TMP256:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP257:%.*]] = inttoptr i32 [[TMP256]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP258:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP257]], i32 -30 -; POST-PROCESS-NEXT: [[TMP259:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP258]], i32 0, i32 0, i64 50 -; POST-PROCESS-NEXT: store i32 [[TMP133]], ptr addrspace(21) [[TMP259]], align 4 -; POST-PROCESS-NEXT: [[TMP260:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP261:%.*]] = inttoptr i32 [[TMP260]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP262:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP261]], i32 -30 -; POST-PROCESS-NEXT: [[TMP263:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP262]], i32 0, i32 0, i64 51 -; POST-PROCESS-NEXT: store i32 [[TMP138]], ptr addrspace(21) [[TMP263]], align 4 -; POST-PROCESS-NEXT: [[TMP264:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP265:%.*]] = inttoptr i32 [[TMP264]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP266:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP265]], i32 -30 -; POST-PROCESS-NEXT: [[TMP267:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP266]], i32 0, i32 0, i64 52 -; POST-PROCESS-NEXT: store i32 [[TMP143]], ptr addrspace(21) [[TMP267]], align 4 -; POST-PROCESS-NEXT: [[TMP268:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP269:%.*]] = inttoptr i32 [[TMP268]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP270:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP269]], i32 -30 -; POST-PROCESS-NEXT: [[TMP271:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP270]], i32 0, i32 0, i64 53 -; POST-PROCESS-NEXT: store i32 [[TMP148]], ptr addrspace(21) [[TMP271]], align 4 -; POST-PROCESS-NEXT: [[TMP272:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP273:%.*]] = inttoptr i32 [[TMP272]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP274:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP273]], i32 -30 -; POST-PROCESS-NEXT: [[TMP275:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP274]], i32 0, i32 0, i64 54 -; POST-PROCESS-NEXT: store i32 [[TMP153]], ptr addrspace(21) [[TMP275]], align 4 -; POST-PROCESS-NEXT: [[TMP276:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP277:%.*]] = inttoptr i32 [[TMP276]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP278:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP277]], i32 -30 -; POST-PROCESS-NEXT: [[TMP279:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP278]], i32 0, i32 0, i64 55 -; POST-PROCESS-NEXT: store i32 [[TMP158]], ptr addrspace(21) [[TMP279]], align 4 -; POST-PROCESS-NEXT: [[TMP280:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP281:%.*]] = inttoptr i32 [[TMP280]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP282:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP281]], i32 -30 -; POST-PROCESS-NEXT: [[TMP283:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP282]], i32 0, i32 0, i64 56 -; POST-PROCESS-NEXT: store i32 [[TMP163]], ptr addrspace(21) [[TMP283]], align 4 -; POST-PROCESS-NEXT: [[TMP284:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP285:%.*]] = add i32 [[TMP284]], 16 -; POST-PROCESS-NEXT: store i32 [[TMP285]], ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP5]], ptr addrspace(20) addrspacecast (ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT:%.*]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i32 7) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP6]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 8) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP7]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 9) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP8]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 10) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP9]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 11) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP10]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 12) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP11]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 13) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP12]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 14) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP13]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 15) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP14]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 16) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP15]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 17) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP16]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 18) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP17]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 19) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP18]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 20) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP19]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 21) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP20]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 22) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP21]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 23) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP22]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 24) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP23]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 25) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP24]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 26) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP25]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 27) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP26]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 28) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP27]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 29) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP175:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP176:%.*]] = inttoptr i32 [[TMP175]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP177:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP176]], i32 -30 +; POST-PROCESS-NEXT: [[TMP178:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP177]], i32 0, i32 0, i64 30 +; POST-PROCESS-NEXT: store i32 [[TMP32]], ptr addrspace(21) [[TMP178]], align 4 +; POST-PROCESS-NEXT: [[TMP179:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP180:%.*]] = inttoptr i32 [[TMP179]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP181:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP180]], i32 -30 +; POST-PROCESS-NEXT: [[TMP182:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP181]], i32 0, i32 0, i64 31 +; POST-PROCESS-NEXT: store i32 [[TMP37]], ptr addrspace(21) [[TMP182]], align 4 +; POST-PROCESS-NEXT: [[TMP183:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP184:%.*]] = inttoptr i32 [[TMP183]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP185:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP184]], i32 -30 +; POST-PROCESS-NEXT: [[TMP186:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP185]], i32 0, i32 0, i64 32 +; POST-PROCESS-NEXT: store i32 [[TMP42]], ptr addrspace(21) [[TMP186]], align 4 +; POST-PROCESS-NEXT: [[TMP187:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP188:%.*]] = inttoptr i32 [[TMP187]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP189:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP188]], i32 -30 +; POST-PROCESS-NEXT: [[TMP190:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP189]], i32 0, i32 0, i64 33 +; POST-PROCESS-NEXT: store i32 [[TMP47]], ptr addrspace(21) [[TMP190]], align 4 +; POST-PROCESS-NEXT: [[TMP191:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP192:%.*]] = inttoptr i32 [[TMP191]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP193:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP192]], i32 -30 +; POST-PROCESS-NEXT: [[TMP194:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP193]], i32 0, i32 0, i64 34 +; POST-PROCESS-NEXT: store i32 [[TMP52]], ptr addrspace(21) [[TMP194]], align 4 +; POST-PROCESS-NEXT: [[TMP195:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP196:%.*]] = inttoptr i32 [[TMP195]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP197:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP196]], i32 -30 +; POST-PROCESS-NEXT: [[TMP198:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP197]], i32 0, i32 0, i64 35 +; POST-PROCESS-NEXT: store i32 [[TMP57]], ptr addrspace(21) [[TMP198]], align 4 +; POST-PROCESS-NEXT: [[TMP199:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP200:%.*]] = inttoptr i32 [[TMP199]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP201:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP200]], i32 -30 +; POST-PROCESS-NEXT: [[TMP202:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP201]], i32 0, i32 0, i64 36 +; POST-PROCESS-NEXT: store i32 [[TMP62]], ptr addrspace(21) [[TMP202]], align 4 +; POST-PROCESS-NEXT: [[TMP203:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP204:%.*]] = inttoptr i32 [[TMP203]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP205:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP204]], i32 -30 +; POST-PROCESS-NEXT: [[TMP206:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP205]], i32 0, i32 0, i64 37 +; POST-PROCESS-NEXT: store i32 [[TMP67]], ptr addrspace(21) [[TMP206]], align 4 +; POST-PROCESS-NEXT: [[TMP207:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP208:%.*]] = inttoptr i32 [[TMP207]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP209:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP208]], i32 -30 +; POST-PROCESS-NEXT: [[TMP210:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP209]], i32 0, i32 0, i64 38 +; POST-PROCESS-NEXT: store i32 [[TMP72]], ptr addrspace(21) [[TMP210]], align 4 +; POST-PROCESS-NEXT: [[TMP211:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP212:%.*]] = inttoptr i32 [[TMP211]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP213:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP212]], i32 -30 +; POST-PROCESS-NEXT: [[TMP214:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP213]], i32 0, i32 0, i64 39 +; POST-PROCESS-NEXT: store i32 [[TMP77]], ptr addrspace(21) [[TMP214]], align 4 +; POST-PROCESS-NEXT: [[TMP215:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP216:%.*]] = inttoptr i32 [[TMP215]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP217:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP216]], i32 -30 +; POST-PROCESS-NEXT: [[TMP218:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP217]], i32 0, i32 0, i64 40 +; POST-PROCESS-NEXT: store i32 [[TMP82]], ptr addrspace(21) [[TMP218]], align 4 +; POST-PROCESS-NEXT: [[TMP219:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP220:%.*]] = inttoptr i32 [[TMP219]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP221:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP220]], i32 -30 +; POST-PROCESS-NEXT: [[TMP222:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP221]], i32 0, i32 0, i64 41 +; POST-PROCESS-NEXT: store i32 [[TMP87]], ptr addrspace(21) [[TMP222]], align 4 +; POST-PROCESS-NEXT: [[TMP223:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP224:%.*]] = inttoptr i32 [[TMP223]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP225:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP224]], i32 -30 +; POST-PROCESS-NEXT: [[TMP226:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP225]], i32 0, i32 0, i64 42 +; POST-PROCESS-NEXT: store i32 [[TMP92]], ptr addrspace(21) [[TMP226]], align 4 +; POST-PROCESS-NEXT: [[TMP227:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP228:%.*]] = inttoptr i32 [[TMP227]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP229:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP228]], i32 -30 +; POST-PROCESS-NEXT: [[TMP230:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP229]], i32 0, i32 0, i64 43 +; POST-PROCESS-NEXT: store i32 [[TMP97]], ptr addrspace(21) [[TMP230]], align 4 +; POST-PROCESS-NEXT: [[TMP231:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP232:%.*]] = inttoptr i32 [[TMP231]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP233:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP232]], i32 -30 +; POST-PROCESS-NEXT: [[TMP234:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP233]], i32 0, i32 0, i64 44 +; POST-PROCESS-NEXT: store i32 [[TMP102]], ptr addrspace(21) [[TMP234]], align 4 +; POST-PROCESS-NEXT: [[TMP235:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP236:%.*]] = inttoptr i32 [[TMP235]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP237:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP236]], i32 -30 +; POST-PROCESS-NEXT: [[TMP238:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP237]], i32 0, i32 0, i64 45 +; POST-PROCESS-NEXT: store i32 [[TMP107]], ptr addrspace(21) [[TMP238]], align 4 +; POST-PROCESS-NEXT: [[TMP239:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP240:%.*]] = inttoptr i32 [[TMP239]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP241:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP240]], i32 -30 +; POST-PROCESS-NEXT: [[TMP242:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP241]], i32 0, i32 0, i64 46 +; POST-PROCESS-NEXT: store i32 [[TMP112]], ptr addrspace(21) [[TMP242]], align 4 +; POST-PROCESS-NEXT: [[TMP243:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP244:%.*]] = inttoptr i32 [[TMP243]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP245:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP244]], i32 -30 +; POST-PROCESS-NEXT: [[TMP246:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP245]], i32 0, i32 0, i64 47 +; POST-PROCESS-NEXT: store i32 [[TMP117]], ptr addrspace(21) [[TMP246]], align 4 +; POST-PROCESS-NEXT: [[TMP247:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP248:%.*]] = inttoptr i32 [[TMP247]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP249:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP248]], i32 -30 +; POST-PROCESS-NEXT: [[TMP250:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP249]], i32 0, i32 0, i64 48 +; POST-PROCESS-NEXT: store i32 [[TMP122]], ptr addrspace(21) [[TMP250]], align 4 +; POST-PROCESS-NEXT: [[TMP251:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP252:%.*]] = inttoptr i32 [[TMP251]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP253:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP252]], i32 -30 +; POST-PROCESS-NEXT: [[TMP254:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP253]], i32 0, i32 0, i64 49 +; POST-PROCESS-NEXT: store i32 [[TMP127]], ptr addrspace(21) [[TMP254]], align 4 +; POST-PROCESS-NEXT: [[TMP255:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP256:%.*]] = inttoptr i32 [[TMP255]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP257:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP256]], i32 -30 +; POST-PROCESS-NEXT: [[TMP258:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP257]], i32 0, i32 0, i64 50 +; POST-PROCESS-NEXT: store i32 [[TMP132]], ptr addrspace(21) [[TMP258]], align 4 +; POST-PROCESS-NEXT: [[TMP259:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP260:%.*]] = inttoptr i32 [[TMP259]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP261:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP260]], i32 -30 +; POST-PROCESS-NEXT: [[TMP262:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP261]], i32 0, i32 0, i64 51 +; POST-PROCESS-NEXT: store i32 [[TMP137]], ptr addrspace(21) [[TMP262]], align 4 +; POST-PROCESS-NEXT: [[TMP263:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP264:%.*]] = inttoptr i32 [[TMP263]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP265:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP264]], i32 -30 +; POST-PROCESS-NEXT: [[TMP266:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP265]], i32 0, i32 0, i64 52 +; POST-PROCESS-NEXT: store i32 [[TMP142]], ptr addrspace(21) [[TMP266]], align 4 +; POST-PROCESS-NEXT: [[TMP267:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP268:%.*]] = inttoptr i32 [[TMP267]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP269:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP268]], i32 -30 +; POST-PROCESS-NEXT: [[TMP270:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP269]], i32 0, i32 0, i64 53 +; POST-PROCESS-NEXT: store i32 [[TMP147]], ptr addrspace(21) [[TMP270]], align 4 +; POST-PROCESS-NEXT: [[TMP271:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP272:%.*]] = inttoptr i32 [[TMP271]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP273:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP272]], i32 -30 +; POST-PROCESS-NEXT: [[TMP274:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP273]], i32 0, i32 0, i64 54 +; POST-PROCESS-NEXT: store i32 [[TMP152]], ptr addrspace(21) [[TMP274]], align 4 +; POST-PROCESS-NEXT: [[TMP275:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP276:%.*]] = inttoptr i32 [[TMP275]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP277:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP276]], i32 -30 +; POST-PROCESS-NEXT: [[TMP278:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP277]], i32 0, i32 0, i64 55 +; POST-PROCESS-NEXT: store i32 [[TMP157]], ptr addrspace(21) [[TMP278]], align 4 +; POST-PROCESS-NEXT: [[TMP279:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP280:%.*]] = inttoptr i32 [[TMP279]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP281:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP280]], i32 -30 +; POST-PROCESS-NEXT: [[TMP282:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspace(21) [[TMP281]], i32 0, i32 0, i64 56 +; POST-PROCESS-NEXT: store i32 [[TMP162]], ptr addrspace(21) [[TMP282]], align 4 +; POST-PROCESS-NEXT: [[TMP283:%.*]] = load i32, ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP284:%.*]] = add i32 [[TMP283]], 16 +; POST-PROCESS-NEXT: store i32 [[TMP284]], ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP285:%.*]] = inttoptr i32 [[TMP283]] to ptr addrspace(21) ; POST-PROCESS-NEXT: [[TMP286:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; POST-PROCESS-NEXT: [[TMP287:%.*]] = load i32, ptr [[TMP286]], align 4 -; POST-PROCESS-NEXT: [[TMP288:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP289:%.*]] = inttoptr i32 [[TMP288]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP290:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP289]], i64 -16 -; POST-PROCESS-NEXT: [[TMP291:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP290]], i32 0 -; POST-PROCESS-NEXT: store i32 [[TMP287]], ptr addrspace(21) [[TMP291]], align 4 -; POST-PROCESS-NEXT: [[TMP292:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; POST-PROCESS-NEXT: [[TMP293:%.*]] = load i32, ptr [[TMP292]], align 4 -; POST-PROCESS-NEXT: [[TMP294:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP295:%.*]] = inttoptr i32 [[TMP294]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP296:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP295]], i64 -16 -; POST-PROCESS-NEXT: [[TMP297:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP296]], i32 0 -; POST-PROCESS-NEXT: [[TMP298:%.*]] = getelementptr [4 x i32], ptr addrspace(21) [[TMP297]], i32 0, i32 1 -; POST-PROCESS-NEXT: store i32 [[TMP293]], ptr addrspace(21) [[TMP298]], align 4 -; POST-PROCESS-NEXT: [[TMP299:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 2 -; POST-PROCESS-NEXT: [[TMP300:%.*]] = load i32, ptr [[TMP299]], align 4 -; POST-PROCESS-NEXT: [[TMP301:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP302:%.*]] = inttoptr i32 [[TMP301]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP303:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP302]], i64 -16 -; POST-PROCESS-NEXT: [[TMP304:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP303]], i32 0 -; POST-PROCESS-NEXT: [[TMP305:%.*]] = getelementptr [4 x i32], ptr addrspace(21) [[TMP304]], i32 0, i32 2 -; POST-PROCESS-NEXT: store i32 [[TMP300]], ptr addrspace(21) [[TMP305]], align 4 -; POST-PROCESS-NEXT: [[TMP306:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 3 -; POST-PROCESS-NEXT: [[TMP307:%.*]] = load i32, ptr [[TMP306]], align 4 -; POST-PROCESS-NEXT: [[TMP308:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP309:%.*]] = inttoptr i32 [[TMP308]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP310:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP309]], i64 -16 -; POST-PROCESS-NEXT: [[TMP311:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP310]], i32 0 -; POST-PROCESS-NEXT: [[TMP312:%.*]] = getelementptr [4 x i32], ptr addrspace(21) [[TMP311]], i32 0, i32 3 -; POST-PROCESS-NEXT: store i32 [[TMP307]], ptr addrspace(21) [[TMP312]], align 4 -; POST-PROCESS-NEXT: [[TMP313:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP314:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @ClosestHit.resume.0 to i64)) -; POST-PROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 4, i32 [[TMP313]], i64 [[TMP314]], [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]]), !continuation.registercount !17, !continuation.returnedRegistercount !17 +; POST-PROCESS-NEXT: [[TMP287:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(21) [[TMP285]], i32 0, i32 0 +; POST-PROCESS-NEXT: [[TMP288:%.*]] = load i32, ptr [[TMP286]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP288]], ptr addrspace(21) [[TMP287]], align 4 +; POST-PROCESS-NEXT: [[TMP289:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; POST-PROCESS-NEXT: [[TMP290:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(21) [[TMP285]], i32 0, i32 1 +; POST-PROCESS-NEXT: [[TMP291:%.*]] = load i32, ptr [[TMP289]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP291]], ptr addrspace(21) [[TMP290]], align 4 +; POST-PROCESS-NEXT: [[TMP292:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; POST-PROCESS-NEXT: [[TMP293:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(21) [[TMP285]], i32 0, i32 2 +; POST-PROCESS-NEXT: [[TMP294:%.*]] = load i32, ptr [[TMP292]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP294]], ptr addrspace(21) [[TMP293]], align 4 +; POST-PROCESS-NEXT: [[TMP295:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 3 +; POST-PROCESS-NEXT: [[TMP296:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(21) [[TMP285]], i32 0, i32 3 +; POST-PROCESS-NEXT: [[TMP297:%.*]] = load i32, ptr [[TMP295]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP297]], ptr addrspace(21) [[TMP296]], align 4 +; POST-PROCESS-NEXT: [[TMP298:%.*]] = load i32, ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP299:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @ClosestHit.resume.0 to i64)) +; POST-PROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 4, i32 [[TMP298]], i64 [[TMP299]], [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]]), !continuation.registercount !17, !continuation.returnedRegistercount !17 ; POST-PROCESS-NEXT: unreachable ; ; -; POST-PROCESS-LABEL: define void @ClosestHit.resume.0( +; POST-PROCESS-LABEL: define dso_local void @ClosestHit.resume.0( ; POST-PROCESS-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage !24 !continuation.registercount !17 !continuation !25 { ; POST-PROCESS-NEXT: entryresume.0: ; POST-PROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA]], align 8 @@ -1936,205 +1933,190 @@ attributes #3 = { nounwind } ; POST-PROCESS-NEXT: [[CSP:%.*]] = alloca i32, align 4 ; POST-PROCESS-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], ptr [[SYSTEM_DATA]], align 4 ; POST-PROCESS-NEXT: store i32 [[TMP0]], ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP2:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; POST-PROCESS-NEXT: [[TMP3:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP4:%.*]] = inttoptr i32 [[TMP3]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP4]], i64 -16 -; POST-PROCESS-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP5]], i32 0 -; POST-PROCESS-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(21) [[TMP6]], align 4 -; POST-PROCESS-NEXT: store i32 [[TMP7]], ptr [[TMP2]], align 4 -; POST-PROCESS-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; POST-PROCESS-NEXT: [[TMP9:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP10:%.*]] = inttoptr i32 [[TMP9]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP10]], i64 -16 -; POST-PROCESS-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP11]], i32 0 -; POST-PROCESS-NEXT: [[TMP13:%.*]] = getelementptr [4 x i32], ptr addrspace(21) [[TMP12]], i32 0, i32 1 -; POST-PROCESS-NEXT: [[TMP14:%.*]] = load i32, ptr addrspace(21) [[TMP13]], align 4 -; POST-PROCESS-NEXT: store i32 [[TMP14]], ptr [[TMP8]], align 4 -; POST-PROCESS-NEXT: [[TMP15:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 2 -; POST-PROCESS-NEXT: [[TMP16:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP17:%.*]] = inttoptr i32 [[TMP16]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP17]], i64 -16 -; POST-PROCESS-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP18]], i32 0 -; POST-PROCESS-NEXT: [[TMP20:%.*]] = getelementptr [4 x i32], ptr addrspace(21) [[TMP19]], i32 0, i32 2 -; POST-PROCESS-NEXT: [[TMP21:%.*]] = load i32, ptr addrspace(21) [[TMP20]], align 4 -; POST-PROCESS-NEXT: store i32 [[TMP21]], ptr [[TMP15]], align 4 -; POST-PROCESS-NEXT: [[TMP22:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 3 -; POST-PROCESS-NEXT: [[TMP23:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP24:%.*]] = inttoptr i32 [[TMP23]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP24]], i64 -16 -; POST-PROCESS-NEXT: [[TMP26:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP25]], i32 0 -; POST-PROCESS-NEXT: [[TMP27:%.*]] = getelementptr [4 x i32], ptr addrspace(21) [[TMP26]], i32 0, i32 3 -; POST-PROCESS-NEXT: [[TMP28:%.*]] = load i32, ptr addrspace(21) [[TMP27]], align 4 -; POST-PROCESS-NEXT: store i32 [[TMP28]], ptr [[TMP22]], align 4 -; POST-PROCESS-NEXT: [[TMP29:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP30:%.*]] = add i32 [[TMP29]], -16 -; POST-PROCESS-NEXT: store i32 [[TMP30]], ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP2:%.*]] = load i32, ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP3]], i64 -16 +; POST-PROCESS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 0 +; POST-PROCESS-NEXT: [[TMP6:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; POST-PROCESS-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(21) [[TMP5]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP7]], ptr [[TMP6]], align 4 +; POST-PROCESS-NEXT: [[TMP8:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 1 +; POST-PROCESS-NEXT: [[TMP9:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; POST-PROCESS-NEXT: [[TMP10:%.*]] = load i32, ptr addrspace(21) [[TMP8]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP10]], ptr [[TMP9]], align 4 +; POST-PROCESS-NEXT: [[TMP11:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 2 +; POST-PROCESS-NEXT: [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 2 +; POST-PROCESS-NEXT: [[TMP13:%.*]] = load i32, ptr addrspace(21) [[TMP11]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP13]], ptr [[TMP12]], align 4 +; POST-PROCESS-NEXT: [[TMP14:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 3 +; POST-PROCESS-NEXT: [[TMP15:%.*]] = getelementptr inbounds [4 x i32], ptr [[CONT_STATE]], i32 0, i32 3 +; POST-PROCESS-NEXT: [[TMP16:%.*]] = load i32, ptr addrspace(21) [[TMP14]], align 4 +; POST-PROCESS-NEXT: store i32 [[TMP16]], ptr [[TMP15]], align 4 +; POST-PROCESS-NEXT: [[TMP17:%.*]] = load i32, ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP18:%.*]] = add i32 [[TMP17]], -16 +; POST-PROCESS-NEXT: store i32 [[TMP18]], ptr [[CSP]], align 4 ; POST-PROCESS-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = getelementptr inbounds [[CLOSESTHIT_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 -; POST-PROCESS-NEXT: [[TMP31:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA:%.*]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; POST-PROCESS-NEXT: [[TMP32:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i32 7) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP33:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 8) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP34:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 9) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP35:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 10) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP36:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 11) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP37:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 12) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP38:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 13) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP39:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 14) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP40:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 15) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP41:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 16) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP42:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 17) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP43:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 18) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP44:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 19) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP45:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 20) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP46:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 21) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP47:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 22) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP48:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 23) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP49:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 24) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP50:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 25) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP51:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 26) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP52:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 27) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP53:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 28) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP54:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 29) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP55:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP56:%.*]] = inttoptr i32 [[TMP55]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP57:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP56]], i32 -30 -; POST-PROCESS-NEXT: [[TMP58:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP57]], i32 0, i32 0, i64 30 -; POST-PROCESS-NEXT: [[TMP59:%.*]] = load i32, ptr addrspace(21) [[TMP58]], align 4 -; POST-PROCESS-NEXT: [[TMP60:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP61:%.*]] = inttoptr i32 [[TMP60]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP62:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP61]], i32 -30 -; POST-PROCESS-NEXT: [[TMP63:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP62]], i32 0, i32 0, i64 31 -; POST-PROCESS-NEXT: [[TMP64:%.*]] = load i32, ptr addrspace(21) [[TMP63]], align 4 -; POST-PROCESS-NEXT: [[TMP65:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP66:%.*]] = inttoptr i32 [[TMP65]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP67:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP66]], i32 -30 -; POST-PROCESS-NEXT: [[TMP68:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP67]], i32 0, i32 0, i64 32 -; POST-PROCESS-NEXT: [[TMP69:%.*]] = load i32, ptr addrspace(21) [[TMP68]], align 4 -; POST-PROCESS-NEXT: [[TMP70:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP71:%.*]] = inttoptr i32 [[TMP70]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP72:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP71]], i32 -30 -; POST-PROCESS-NEXT: [[TMP73:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP72]], i32 0, i32 0, i64 33 -; POST-PROCESS-NEXT: [[TMP74:%.*]] = load i32, ptr addrspace(21) [[TMP73]], align 4 -; POST-PROCESS-NEXT: [[TMP75:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP76:%.*]] = inttoptr i32 [[TMP75]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP77:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP76]], i32 -30 -; POST-PROCESS-NEXT: [[TMP78:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP77]], i32 0, i32 0, i64 34 -; POST-PROCESS-NEXT: [[TMP79:%.*]] = load i32, ptr addrspace(21) [[TMP78]], align 4 -; POST-PROCESS-NEXT: [[TMP80:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP81:%.*]] = inttoptr i32 [[TMP80]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP82:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP81]], i32 -30 -; POST-PROCESS-NEXT: [[TMP83:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP82]], i32 0, i32 0, i64 35 -; POST-PROCESS-NEXT: [[TMP84:%.*]] = load i32, ptr addrspace(21) [[TMP83]], align 4 -; POST-PROCESS-NEXT: [[TMP85:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP86:%.*]] = inttoptr i32 [[TMP85]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP87:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP86]], i32 -30 -; POST-PROCESS-NEXT: [[TMP88:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP87]], i32 0, i32 0, i64 36 -; POST-PROCESS-NEXT: [[TMP89:%.*]] = load i32, ptr addrspace(21) [[TMP88]], align 4 -; POST-PROCESS-NEXT: [[TMP90:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP91:%.*]] = inttoptr i32 [[TMP90]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP92:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP91]], i32 -30 -; POST-PROCESS-NEXT: [[TMP93:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP92]], i32 0, i32 0, i64 37 -; POST-PROCESS-NEXT: [[TMP94:%.*]] = load i32, ptr addrspace(21) [[TMP93]], align 4 -; POST-PROCESS-NEXT: [[TMP95:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP96:%.*]] = inttoptr i32 [[TMP95]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP97:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP96]], i32 -30 -; POST-PROCESS-NEXT: [[TMP98:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP97]], i32 0, i32 0, i64 38 -; POST-PROCESS-NEXT: [[TMP99:%.*]] = load i32, ptr addrspace(21) [[TMP98]], align 4 -; POST-PROCESS-NEXT: [[TMP100:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP101:%.*]] = inttoptr i32 [[TMP100]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP102:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP101]], i32 -30 -; POST-PROCESS-NEXT: [[TMP103:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP102]], i32 0, i32 0, i64 39 -; POST-PROCESS-NEXT: [[TMP104:%.*]] = load i32, ptr addrspace(21) [[TMP103]], align 4 -; POST-PROCESS-NEXT: [[TMP105:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP106:%.*]] = inttoptr i32 [[TMP105]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP107:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP106]], i32 -30 -; POST-PROCESS-NEXT: [[TMP108:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP107]], i32 0, i32 0, i64 40 -; POST-PROCESS-NEXT: [[TMP109:%.*]] = load i32, ptr addrspace(21) [[TMP108]], align 4 -; POST-PROCESS-NEXT: [[TMP110:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP111:%.*]] = inttoptr i32 [[TMP110]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP112:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP111]], i32 -30 -; POST-PROCESS-NEXT: [[TMP113:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP112]], i32 0, i32 0, i64 41 -; POST-PROCESS-NEXT: [[TMP114:%.*]] = load i32, ptr addrspace(21) [[TMP113]], align 4 -; POST-PROCESS-NEXT: [[TMP115:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP116:%.*]] = inttoptr i32 [[TMP115]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP117:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP116]], i32 -30 -; POST-PROCESS-NEXT: [[TMP118:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP117]], i32 0, i32 0, i64 42 -; POST-PROCESS-NEXT: [[TMP119:%.*]] = load i32, ptr addrspace(21) [[TMP118]], align 4 -; POST-PROCESS-NEXT: [[TMP120:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP121:%.*]] = inttoptr i32 [[TMP120]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP122:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP121]], i32 -30 -; POST-PROCESS-NEXT: [[TMP123:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP122]], i32 0, i32 0, i64 43 -; POST-PROCESS-NEXT: [[TMP124:%.*]] = load i32, ptr addrspace(21) [[TMP123]], align 4 -; POST-PROCESS-NEXT: [[TMP125:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP126:%.*]] = inttoptr i32 [[TMP125]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP127:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP126]], i32 -30 -; POST-PROCESS-NEXT: [[TMP128:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP127]], i32 0, i32 0, i64 44 -; POST-PROCESS-NEXT: [[TMP129:%.*]] = load i32, ptr addrspace(21) [[TMP128]], align 4 -; POST-PROCESS-NEXT: [[TMP130:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP131:%.*]] = inttoptr i32 [[TMP130]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP132:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP131]], i32 -30 -; POST-PROCESS-NEXT: [[TMP133:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP132]], i32 0, i32 0, i64 45 -; POST-PROCESS-NEXT: [[TMP134:%.*]] = load i32, ptr addrspace(21) [[TMP133]], align 4 -; POST-PROCESS-NEXT: [[TMP135:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP136:%.*]] = inttoptr i32 [[TMP135]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP137:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP136]], i32 -30 -; POST-PROCESS-NEXT: [[TMP138:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP137]], i32 0, i32 0, i64 46 -; POST-PROCESS-NEXT: [[TMP139:%.*]] = load i32, ptr addrspace(21) [[TMP138]], align 4 -; POST-PROCESS-NEXT: [[TMP140:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP141:%.*]] = inttoptr i32 [[TMP140]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP142:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP141]], i32 -30 -; POST-PROCESS-NEXT: [[TMP143:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP142]], i32 0, i32 0, i64 47 -; POST-PROCESS-NEXT: [[TMP144:%.*]] = load i32, ptr addrspace(21) [[TMP143]], align 4 -; POST-PROCESS-NEXT: [[TMP145:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP146:%.*]] = inttoptr i32 [[TMP145]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP147:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP146]], i32 -30 -; POST-PROCESS-NEXT: [[TMP148:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP147]], i32 0, i32 0, i64 48 -; POST-PROCESS-NEXT: [[TMP149:%.*]] = load i32, ptr addrspace(21) [[TMP148]], align 4 -; POST-PROCESS-NEXT: [[TMP150:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP151:%.*]] = inttoptr i32 [[TMP150]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP152:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP151]], i32 -30 -; POST-PROCESS-NEXT: [[TMP153:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP152]], i32 0, i32 0, i64 49 -; POST-PROCESS-NEXT: [[TMP154:%.*]] = load i32, ptr addrspace(21) [[TMP153]], align 4 -; POST-PROCESS-NEXT: [[TMP155:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP156:%.*]] = inttoptr i32 [[TMP155]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP157:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP156]], i32 -30 -; POST-PROCESS-NEXT: [[TMP158:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP157]], i32 0, i32 0, i64 50 -; POST-PROCESS-NEXT: [[TMP159:%.*]] = load i32, ptr addrspace(21) [[TMP158]], align 4 -; POST-PROCESS-NEXT: [[TMP160:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP161:%.*]] = inttoptr i32 [[TMP160]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP162:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP161]], i32 -30 -; POST-PROCESS-NEXT: [[TMP163:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP162]], i32 0, i32 0, i64 51 -; POST-PROCESS-NEXT: [[TMP164:%.*]] = load i32, ptr addrspace(21) [[TMP163]], align 4 -; POST-PROCESS-NEXT: [[TMP165:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP166:%.*]] = inttoptr i32 [[TMP165]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP167:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP166]], i32 -30 -; POST-PROCESS-NEXT: [[TMP168:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP167]], i32 0, i32 0, i64 52 -; POST-PROCESS-NEXT: [[TMP169:%.*]] = load i32, ptr addrspace(21) [[TMP168]], align 4 -; POST-PROCESS-NEXT: [[TMP170:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP171:%.*]] = inttoptr i32 [[TMP170]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP172:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP171]], i32 -30 -; POST-PROCESS-NEXT: [[TMP173:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP172]], i32 0, i32 0, i64 53 -; POST-PROCESS-NEXT: [[TMP174:%.*]] = load i32, ptr addrspace(21) [[TMP173]], align 4 -; POST-PROCESS-NEXT: [[TMP175:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP176:%.*]] = inttoptr i32 [[TMP175]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP177:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP176]], i32 -30 -; POST-PROCESS-NEXT: [[TMP178:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP177]], i32 0, i32 0, i64 54 -; POST-PROCESS-NEXT: [[TMP179:%.*]] = load i32, ptr addrspace(21) [[TMP178]], align 4 -; POST-PROCESS-NEXT: [[TMP180:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP181:%.*]] = inttoptr i32 [[TMP180]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP182:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP181]], i32 -30 -; POST-PROCESS-NEXT: [[TMP183:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP182]], i32 0, i32 0, i64 55 -; POST-PROCESS-NEXT: [[TMP184:%.*]] = load i32, ptr addrspace(21) [[TMP183]], align 4 -; POST-PROCESS-NEXT: [[TMP185:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP186:%.*]] = inttoptr i32 [[TMP185]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP187:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP186]], i32 -30 -; POST-PROCESS-NEXT: [[TMP188:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP187]], i32 0, i32 0, i64 56 -; POST-PROCESS-NEXT: [[TMP189:%.*]] = load i32, ptr addrspace(21) [[TMP188]], align 4 -; POST-PROCESS-NEXT: [[TMP190:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP191:%.*]] = add i32 [[TMP190]], -108 -; POST-PROCESS-NEXT: store i32 [[TMP191]], ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP19:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA:%.*]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; POST-PROCESS-NEXT: [[TMP20:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i32 7) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP21:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 8) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP22:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 9) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP23:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 10) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP24:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 11) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP25:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 12) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP26:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 13) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP27:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 14) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP28:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 15) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP29:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 16) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP30:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 17) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP31:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 18) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP32:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 19) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP33:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 20) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP34:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 21) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP35:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 22) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP36:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 23) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP37:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 24) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP38:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 25) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP39:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 26) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP40:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 27) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP41:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 28) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP42:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 29) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP43:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP44:%.*]] = inttoptr i32 [[TMP43]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP45:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP44]], i32 -30 +; POST-PROCESS-NEXT: [[TMP46:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP45]], i32 0, i32 0, i64 30 +; POST-PROCESS-NEXT: [[TMP47:%.*]] = load i32, ptr addrspace(21) [[TMP46]], align 4 +; POST-PROCESS-NEXT: [[TMP48:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP49:%.*]] = inttoptr i32 [[TMP48]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP50:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP49]], i32 -30 +; POST-PROCESS-NEXT: [[TMP51:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP50]], i32 0, i32 0, i64 31 +; POST-PROCESS-NEXT: [[TMP52:%.*]] = load i32, ptr addrspace(21) [[TMP51]], align 4 +; POST-PROCESS-NEXT: [[TMP53:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP54:%.*]] = inttoptr i32 [[TMP53]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP55:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP54]], i32 -30 +; POST-PROCESS-NEXT: [[TMP56:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP55]], i32 0, i32 0, i64 32 +; POST-PROCESS-NEXT: [[TMP57:%.*]] = load i32, ptr addrspace(21) [[TMP56]], align 4 +; POST-PROCESS-NEXT: [[TMP58:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP59:%.*]] = inttoptr i32 [[TMP58]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP60:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP59]], i32 -30 +; POST-PROCESS-NEXT: [[TMP61:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP60]], i32 0, i32 0, i64 33 +; POST-PROCESS-NEXT: [[TMP62:%.*]] = load i32, ptr addrspace(21) [[TMP61]], align 4 +; POST-PROCESS-NEXT: [[TMP63:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP64:%.*]] = inttoptr i32 [[TMP63]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP65:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP64]], i32 -30 +; POST-PROCESS-NEXT: [[TMP66:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP65]], i32 0, i32 0, i64 34 +; POST-PROCESS-NEXT: [[TMP67:%.*]] = load i32, ptr addrspace(21) [[TMP66]], align 4 +; POST-PROCESS-NEXT: [[TMP68:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP69:%.*]] = inttoptr i32 [[TMP68]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP70:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP69]], i32 -30 +; POST-PROCESS-NEXT: [[TMP71:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP70]], i32 0, i32 0, i64 35 +; POST-PROCESS-NEXT: [[TMP72:%.*]] = load i32, ptr addrspace(21) [[TMP71]], align 4 +; POST-PROCESS-NEXT: [[TMP73:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP74:%.*]] = inttoptr i32 [[TMP73]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP75:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP74]], i32 -30 +; POST-PROCESS-NEXT: [[TMP76:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP75]], i32 0, i32 0, i64 36 +; POST-PROCESS-NEXT: [[TMP77:%.*]] = load i32, ptr addrspace(21) [[TMP76]], align 4 +; POST-PROCESS-NEXT: [[TMP78:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP79:%.*]] = inttoptr i32 [[TMP78]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP80:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP79]], i32 -30 +; POST-PROCESS-NEXT: [[TMP81:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP80]], i32 0, i32 0, i64 37 +; POST-PROCESS-NEXT: [[TMP82:%.*]] = load i32, ptr addrspace(21) [[TMP81]], align 4 +; POST-PROCESS-NEXT: [[TMP83:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP84:%.*]] = inttoptr i32 [[TMP83]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP85:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP84]], i32 -30 +; POST-PROCESS-NEXT: [[TMP86:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP85]], i32 0, i32 0, i64 38 +; POST-PROCESS-NEXT: [[TMP87:%.*]] = load i32, ptr addrspace(21) [[TMP86]], align 4 +; POST-PROCESS-NEXT: [[TMP88:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP89:%.*]] = inttoptr i32 [[TMP88]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP90:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP89]], i32 -30 +; POST-PROCESS-NEXT: [[TMP91:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP90]], i32 0, i32 0, i64 39 +; POST-PROCESS-NEXT: [[TMP92:%.*]] = load i32, ptr addrspace(21) [[TMP91]], align 4 +; POST-PROCESS-NEXT: [[TMP93:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP94:%.*]] = inttoptr i32 [[TMP93]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP95:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP94]], i32 -30 +; POST-PROCESS-NEXT: [[TMP96:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP95]], i32 0, i32 0, i64 40 +; POST-PROCESS-NEXT: [[TMP97:%.*]] = load i32, ptr addrspace(21) [[TMP96]], align 4 +; POST-PROCESS-NEXT: [[TMP98:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP99:%.*]] = inttoptr i32 [[TMP98]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP100:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP99]], i32 -30 +; POST-PROCESS-NEXT: [[TMP101:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP100]], i32 0, i32 0, i64 41 +; POST-PROCESS-NEXT: [[TMP102:%.*]] = load i32, ptr addrspace(21) [[TMP101]], align 4 +; POST-PROCESS-NEXT: [[TMP103:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP104:%.*]] = inttoptr i32 [[TMP103]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP105:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP104]], i32 -30 +; POST-PROCESS-NEXT: [[TMP106:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP105]], i32 0, i32 0, i64 42 +; POST-PROCESS-NEXT: [[TMP107:%.*]] = load i32, ptr addrspace(21) [[TMP106]], align 4 +; POST-PROCESS-NEXT: [[TMP108:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP109:%.*]] = inttoptr i32 [[TMP108]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP110:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP109]], i32 -30 +; POST-PROCESS-NEXT: [[TMP111:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP110]], i32 0, i32 0, i64 43 +; POST-PROCESS-NEXT: [[TMP112:%.*]] = load i32, ptr addrspace(21) [[TMP111]], align 4 +; POST-PROCESS-NEXT: [[TMP113:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP114:%.*]] = inttoptr i32 [[TMP113]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP115:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP114]], i32 -30 +; POST-PROCESS-NEXT: [[TMP116:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP115]], i32 0, i32 0, i64 44 +; POST-PROCESS-NEXT: [[TMP117:%.*]] = load i32, ptr addrspace(21) [[TMP116]], align 4 +; POST-PROCESS-NEXT: [[TMP118:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP119:%.*]] = inttoptr i32 [[TMP118]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP120:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP119]], i32 -30 +; POST-PROCESS-NEXT: [[TMP121:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP120]], i32 0, i32 0, i64 45 +; POST-PROCESS-NEXT: [[TMP122:%.*]] = load i32, ptr addrspace(21) [[TMP121]], align 4 +; POST-PROCESS-NEXT: [[TMP123:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP124:%.*]] = inttoptr i32 [[TMP123]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP125:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP124]], i32 -30 +; POST-PROCESS-NEXT: [[TMP126:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP125]], i32 0, i32 0, i64 46 +; POST-PROCESS-NEXT: [[TMP127:%.*]] = load i32, ptr addrspace(21) [[TMP126]], align 4 +; POST-PROCESS-NEXT: [[TMP128:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP129:%.*]] = inttoptr i32 [[TMP128]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP130:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP129]], i32 -30 +; POST-PROCESS-NEXT: [[TMP131:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP130]], i32 0, i32 0, i64 47 +; POST-PROCESS-NEXT: [[TMP132:%.*]] = load i32, ptr addrspace(21) [[TMP131]], align 4 +; POST-PROCESS-NEXT: [[TMP133:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP134:%.*]] = inttoptr i32 [[TMP133]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP135:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP134]], i32 -30 +; POST-PROCESS-NEXT: [[TMP136:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP135]], i32 0, i32 0, i64 48 +; POST-PROCESS-NEXT: [[TMP137:%.*]] = load i32, ptr addrspace(21) [[TMP136]], align 4 +; POST-PROCESS-NEXT: [[TMP138:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP139:%.*]] = inttoptr i32 [[TMP138]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP140:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP139]], i32 -30 +; POST-PROCESS-NEXT: [[TMP141:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP140]], i32 0, i32 0, i64 49 +; POST-PROCESS-NEXT: [[TMP142:%.*]] = load i32, ptr addrspace(21) [[TMP141]], align 4 +; POST-PROCESS-NEXT: [[TMP143:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP144:%.*]] = inttoptr i32 [[TMP143]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP145:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP144]], i32 -30 +; POST-PROCESS-NEXT: [[TMP146:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP145]], i32 0, i32 0, i64 50 +; POST-PROCESS-NEXT: [[TMP147:%.*]] = load i32, ptr addrspace(21) [[TMP146]], align 4 +; POST-PROCESS-NEXT: [[TMP148:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP149:%.*]] = inttoptr i32 [[TMP148]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP150:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP149]], i32 -30 +; POST-PROCESS-NEXT: [[TMP151:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP150]], i32 0, i32 0, i64 51 +; POST-PROCESS-NEXT: [[TMP152:%.*]] = load i32, ptr addrspace(21) [[TMP151]], align 4 +; POST-PROCESS-NEXT: [[TMP153:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP154:%.*]] = inttoptr i32 [[TMP153]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP155:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP154]], i32 -30 +; POST-PROCESS-NEXT: [[TMP156:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP155]], i32 0, i32 0, i64 52 +; POST-PROCESS-NEXT: [[TMP157:%.*]] = load i32, ptr addrspace(21) [[TMP156]], align 4 +; POST-PROCESS-NEXT: [[TMP158:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP159:%.*]] = inttoptr i32 [[TMP158]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP160:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP159]], i32 -30 +; POST-PROCESS-NEXT: [[TMP161:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP160]], i32 0, i32 0, i64 53 +; POST-PROCESS-NEXT: [[TMP162:%.*]] = load i32, ptr addrspace(21) [[TMP161]], align 4 +; POST-PROCESS-NEXT: [[TMP163:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP164:%.*]] = inttoptr i32 [[TMP163]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP165:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP164]], i32 -30 +; POST-PROCESS-NEXT: [[TMP166:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP165]], i32 0, i32 0, i64 54 +; POST-PROCESS-NEXT: [[TMP167:%.*]] = load i32, ptr addrspace(21) [[TMP166]], align 4 +; POST-PROCESS-NEXT: [[TMP168:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP169:%.*]] = inttoptr i32 [[TMP168]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP170:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP169]], i32 -30 +; POST-PROCESS-NEXT: [[TMP171:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP170]], i32 0, i32 0, i64 55 +; POST-PROCESS-NEXT: [[TMP172:%.*]] = load i32, ptr addrspace(21) [[TMP171]], align 4 +; POST-PROCESS-NEXT: [[TMP173:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP174:%.*]] = inttoptr i32 [[TMP173]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP175:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP174]], i32 -30 +; POST-PROCESS-NEXT: [[TMP176:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP175]], i32 0, i32 0, i64 56 +; POST-PROCESS-NEXT: [[TMP177:%.*]] = load i32, ptr addrspace(21) [[TMP176]], align 4 ; POST-PROCESS-NEXT: [[DOTFCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], 0 -; POST-PROCESS-NEXT: [[DOTFCA_0_GEP51:%.*]] = getelementptr inbounds [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[TMP31]], i32 0, i32 0 +; POST-PROCESS-NEXT: [[DOTFCA_0_GEP51:%.*]] = getelementptr inbounds [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[TMP19]], i32 0, i32 0 ; POST-PROCESS-NEXT: store i32 [[DOTFCA_0_EXTRACT]], ptr [[DOTFCA_0_GEP51]], align 4 ; POST-PROCESS-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; POST-PROCESS-NEXT: [[DOTRELOAD_ADDR:%.*]] = getelementptr inbounds [[CLOSESTHIT_FRAME]], ptr [[CONT_STATE]], i32 0, i32 2 @@ -2142,173 +2124,172 @@ attributes #3 = { nounwind } ; POST-PROCESS-NEXT: [[RETURNADDR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[CLOSESTHIT_FRAME]], ptr [[CONT_STATE]], i32 0, i32 1 ; POST-PROCESS-NEXT: [[RETURNADDR_RELOAD:%.*]] = load i64, ptr [[RETURNADDR_RELOAD_ADDR]], align 4 ; POST-PROCESS-NEXT: store i32 [[DOTRELOAD]], ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: store i32 [[TMP32]], ptr addrspace(20) addrspacecast (ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i32 7) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP33]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 8) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP34]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 9) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP35]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 10) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP36]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 11) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP37]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 12) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP38]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 13) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP39]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 14) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP40]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 15) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP41]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 16) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP42]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 17) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP43]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 18) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP44]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 19) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP45]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 20) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP46]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 21) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP47]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 22) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP48]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 23) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP49]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 24) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP50]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 25) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP51]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 26) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP52]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 27) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP53]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 28) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: store i32 [[TMP54]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 29) to ptr addrspace(20)), align 4 -; POST-PROCESS-NEXT: [[TMP192:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP193:%.*]] = inttoptr i32 [[TMP192]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP194:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP193]], i32 -30 -; POST-PROCESS-NEXT: [[TMP195:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP194]], i32 0, i32 0, i64 30 -; POST-PROCESS-NEXT: store i32 [[TMP59]], ptr addrspace(21) [[TMP195]], align 4 -; POST-PROCESS-NEXT: [[TMP196:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP197:%.*]] = inttoptr i32 [[TMP196]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP198:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP197]], i32 -30 -; POST-PROCESS-NEXT: [[TMP199:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP198]], i32 0, i32 0, i64 31 -; POST-PROCESS-NEXT: store i32 [[TMP64]], ptr addrspace(21) [[TMP199]], align 4 -; POST-PROCESS-NEXT: [[TMP200:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP201:%.*]] = inttoptr i32 [[TMP200]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP202:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP201]], i32 -30 -; POST-PROCESS-NEXT: [[TMP203:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP202]], i32 0, i32 0, i64 32 -; POST-PROCESS-NEXT: store i32 [[TMP69]], ptr addrspace(21) [[TMP203]], align 4 -; POST-PROCESS-NEXT: [[TMP204:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP205:%.*]] = inttoptr i32 [[TMP204]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP206:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP205]], i32 -30 -; POST-PROCESS-NEXT: [[TMP207:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP206]], i32 0, i32 0, i64 33 -; POST-PROCESS-NEXT: store i32 [[TMP74]], ptr addrspace(21) [[TMP207]], align 4 -; POST-PROCESS-NEXT: [[TMP208:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP209:%.*]] = inttoptr i32 [[TMP208]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP210:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP209]], i32 -30 -; POST-PROCESS-NEXT: [[TMP211:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP210]], i32 0, i32 0, i64 34 -; POST-PROCESS-NEXT: store i32 [[TMP79]], ptr addrspace(21) [[TMP211]], align 4 -; POST-PROCESS-NEXT: [[TMP212:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP213:%.*]] = inttoptr i32 [[TMP212]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP214:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP213]], i32 -30 -; POST-PROCESS-NEXT: [[TMP215:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP214]], i32 0, i32 0, i64 35 -; POST-PROCESS-NEXT: store i32 [[TMP84]], ptr addrspace(21) [[TMP215]], align 4 -; POST-PROCESS-NEXT: [[TMP216:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP217:%.*]] = inttoptr i32 [[TMP216]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP218:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP217]], i32 -30 -; POST-PROCESS-NEXT: [[TMP219:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP218]], i32 0, i32 0, i64 36 -; POST-PROCESS-NEXT: store i32 [[TMP89]], ptr addrspace(21) [[TMP219]], align 4 -; POST-PROCESS-NEXT: [[TMP220:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP221:%.*]] = inttoptr i32 [[TMP220]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP222:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP221]], i32 -30 -; POST-PROCESS-NEXT: [[TMP223:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP222]], i32 0, i32 0, i64 37 -; POST-PROCESS-NEXT: store i32 [[TMP94]], ptr addrspace(21) [[TMP223]], align 4 -; POST-PROCESS-NEXT: [[TMP224:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP225:%.*]] = inttoptr i32 [[TMP224]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP226:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP225]], i32 -30 -; POST-PROCESS-NEXT: [[TMP227:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP226]], i32 0, i32 0, i64 38 -; POST-PROCESS-NEXT: store i32 [[TMP99]], ptr addrspace(21) [[TMP227]], align 4 -; POST-PROCESS-NEXT: [[TMP228:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP229:%.*]] = inttoptr i32 [[TMP228]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP230:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP229]], i32 -30 -; POST-PROCESS-NEXT: [[TMP231:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP230]], i32 0, i32 0, i64 39 -; POST-PROCESS-NEXT: store i32 [[TMP104]], ptr addrspace(21) [[TMP231]], align 4 -; POST-PROCESS-NEXT: [[TMP232:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP233:%.*]] = inttoptr i32 [[TMP232]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP234:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP233]], i32 -30 -; POST-PROCESS-NEXT: [[TMP235:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP234]], i32 0, i32 0, i64 40 -; POST-PROCESS-NEXT: store i32 [[TMP109]], ptr addrspace(21) [[TMP235]], align 4 -; POST-PROCESS-NEXT: [[TMP236:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP237:%.*]] = inttoptr i32 [[TMP236]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP238:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP237]], i32 -30 -; POST-PROCESS-NEXT: [[TMP239:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP238]], i32 0, i32 0, i64 41 -; POST-PROCESS-NEXT: store i32 [[TMP114]], ptr addrspace(21) [[TMP239]], align 4 -; POST-PROCESS-NEXT: [[TMP240:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP241:%.*]] = inttoptr i32 [[TMP240]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP242:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP241]], i32 -30 -; POST-PROCESS-NEXT: [[TMP243:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP242]], i32 0, i32 0, i64 42 -; POST-PROCESS-NEXT: store i32 [[TMP119]], ptr addrspace(21) [[TMP243]], align 4 -; POST-PROCESS-NEXT: [[TMP244:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP245:%.*]] = inttoptr i32 [[TMP244]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP246:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP245]], i32 -30 -; POST-PROCESS-NEXT: [[TMP247:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP246]], i32 0, i32 0, i64 43 -; POST-PROCESS-NEXT: store i32 [[TMP124]], ptr addrspace(21) [[TMP247]], align 4 -; POST-PROCESS-NEXT: [[TMP248:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP249:%.*]] = inttoptr i32 [[TMP248]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP250:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP249]], i32 -30 -; POST-PROCESS-NEXT: [[TMP251:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP250]], i32 0, i32 0, i64 44 -; POST-PROCESS-NEXT: store i32 [[TMP129]], ptr addrspace(21) [[TMP251]], align 4 -; POST-PROCESS-NEXT: [[TMP252:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP253:%.*]] = inttoptr i32 [[TMP252]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP254:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP253]], i32 -30 -; POST-PROCESS-NEXT: [[TMP255:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP254]], i32 0, i32 0, i64 45 -; POST-PROCESS-NEXT: store i32 [[TMP134]], ptr addrspace(21) [[TMP255]], align 4 -; POST-PROCESS-NEXT: [[TMP256:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP257:%.*]] = inttoptr i32 [[TMP256]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP258:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP257]], i32 -30 -; POST-PROCESS-NEXT: [[TMP259:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP258]], i32 0, i32 0, i64 46 -; POST-PROCESS-NEXT: store i32 [[TMP139]], ptr addrspace(21) [[TMP259]], align 4 -; POST-PROCESS-NEXT: [[TMP260:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP261:%.*]] = inttoptr i32 [[TMP260]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP262:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP261]], i32 -30 -; POST-PROCESS-NEXT: [[TMP263:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP262]], i32 0, i32 0, i64 47 -; POST-PROCESS-NEXT: store i32 [[TMP144]], ptr addrspace(21) [[TMP263]], align 4 -; POST-PROCESS-NEXT: [[TMP264:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP265:%.*]] = inttoptr i32 [[TMP264]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP266:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP265]], i32 -30 -; POST-PROCESS-NEXT: [[TMP267:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP266]], i32 0, i32 0, i64 48 -; POST-PROCESS-NEXT: store i32 [[TMP149]], ptr addrspace(21) [[TMP267]], align 4 -; POST-PROCESS-NEXT: [[TMP268:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP269:%.*]] = inttoptr i32 [[TMP268]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP270:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP269]], i32 -30 -; POST-PROCESS-NEXT: [[TMP271:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP270]], i32 0, i32 0, i64 49 -; POST-PROCESS-NEXT: store i32 [[TMP154]], ptr addrspace(21) [[TMP271]], align 4 -; POST-PROCESS-NEXT: [[TMP272:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP273:%.*]] = inttoptr i32 [[TMP272]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP274:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP273]], i32 -30 -; POST-PROCESS-NEXT: [[TMP275:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP274]], i32 0, i32 0, i64 50 -; POST-PROCESS-NEXT: store i32 [[TMP159]], ptr addrspace(21) [[TMP275]], align 4 -; POST-PROCESS-NEXT: [[TMP276:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP277:%.*]] = inttoptr i32 [[TMP276]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP278:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP277]], i32 -30 -; POST-PROCESS-NEXT: [[TMP279:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP278]], i32 0, i32 0, i64 51 -; POST-PROCESS-NEXT: store i32 [[TMP164]], ptr addrspace(21) [[TMP279]], align 4 -; POST-PROCESS-NEXT: [[TMP280:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP281:%.*]] = inttoptr i32 [[TMP280]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP282:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP281]], i32 -30 -; POST-PROCESS-NEXT: [[TMP283:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP282]], i32 0, i32 0, i64 52 -; POST-PROCESS-NEXT: store i32 [[TMP169]], ptr addrspace(21) [[TMP283]], align 4 -; POST-PROCESS-NEXT: [[TMP284:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP285:%.*]] = inttoptr i32 [[TMP284]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP286:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP285]], i32 -30 -; POST-PROCESS-NEXT: [[TMP287:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP286]], i32 0, i32 0, i64 53 -; POST-PROCESS-NEXT: store i32 [[TMP174]], ptr addrspace(21) [[TMP287]], align 4 -; POST-PROCESS-NEXT: [[TMP288:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP289:%.*]] = inttoptr i32 [[TMP288]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP290:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP289]], i32 -30 -; POST-PROCESS-NEXT: [[TMP291:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP290]], i32 0, i32 0, i64 54 -; POST-PROCESS-NEXT: store i32 [[TMP179]], ptr addrspace(21) [[TMP291]], align 4 -; POST-PROCESS-NEXT: [[TMP292:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP293:%.*]] = inttoptr i32 [[TMP292]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP294:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP293]], i32 -30 -; POST-PROCESS-NEXT: [[TMP295:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP294]], i32 0, i32 0, i64 55 -; POST-PROCESS-NEXT: store i32 [[TMP184]], ptr addrspace(21) [[TMP295]], align 4 -; POST-PROCESS-NEXT: [[TMP296:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; POST-PROCESS-NEXT: [[TMP297:%.*]] = inttoptr i32 [[TMP296]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP298:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP297]], i32 -30 -; POST-PROCESS-NEXT: [[TMP299:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP298]], i32 0, i32 0, i64 56 -; POST-PROCESS-NEXT: store i32 [[TMP189]], ptr addrspace(21) [[TMP299]], align 4 -; POST-PROCESS-NEXT: [[TMP300:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; POST-PROCESS-NEXT: [[DOTFCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[TMP300]], i32 0, i32 0 +; POST-PROCESS-NEXT: store i32 [[TMP20]], ptr addrspace(20) addrspacecast (ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i32 7) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP21]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 8) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP22]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 9) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP23]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 10) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP24]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 11) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP25]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 12) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP26]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 13) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP27]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 14) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP28]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 15) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP29]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 16) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP30]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 17) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP31]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 18) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP32]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 19) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP33]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 20) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP34]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 21) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP35]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 22) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP36]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 23) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP37]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 24) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP38]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 25) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP39]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 26) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP40]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 27) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP41]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 28) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: store i32 [[TMP42]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 29) to ptr addrspace(20)), align 4 +; POST-PROCESS-NEXT: [[TMP178:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP179:%.*]] = inttoptr i32 [[TMP178]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP180:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP179]], i32 -30 +; POST-PROCESS-NEXT: [[TMP181:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP180]], i32 0, i32 0, i64 30 +; POST-PROCESS-NEXT: store i32 [[TMP47]], ptr addrspace(21) [[TMP181]], align 4 +; POST-PROCESS-NEXT: [[TMP182:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP183:%.*]] = inttoptr i32 [[TMP182]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP184:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP183]], i32 -30 +; POST-PROCESS-NEXT: [[TMP185:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP184]], i32 0, i32 0, i64 31 +; POST-PROCESS-NEXT: store i32 [[TMP52]], ptr addrspace(21) [[TMP185]], align 4 +; POST-PROCESS-NEXT: [[TMP186:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP187:%.*]] = inttoptr i32 [[TMP186]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP188:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP187]], i32 -30 +; POST-PROCESS-NEXT: [[TMP189:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP188]], i32 0, i32 0, i64 32 +; POST-PROCESS-NEXT: store i32 [[TMP57]], ptr addrspace(21) [[TMP189]], align 4 +; POST-PROCESS-NEXT: [[TMP190:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP191:%.*]] = inttoptr i32 [[TMP190]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP192:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP191]], i32 -30 +; POST-PROCESS-NEXT: [[TMP193:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP192]], i32 0, i32 0, i64 33 +; POST-PROCESS-NEXT: store i32 [[TMP62]], ptr addrspace(21) [[TMP193]], align 4 +; POST-PROCESS-NEXT: [[TMP194:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP195:%.*]] = inttoptr i32 [[TMP194]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP196:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP195]], i32 -30 +; POST-PROCESS-NEXT: [[TMP197:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP196]], i32 0, i32 0, i64 34 +; POST-PROCESS-NEXT: store i32 [[TMP67]], ptr addrspace(21) [[TMP197]], align 4 +; POST-PROCESS-NEXT: [[TMP198:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP199:%.*]] = inttoptr i32 [[TMP198]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP200:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP199]], i32 -30 +; POST-PROCESS-NEXT: [[TMP201:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP200]], i32 0, i32 0, i64 35 +; POST-PROCESS-NEXT: store i32 [[TMP72]], ptr addrspace(21) [[TMP201]], align 4 +; POST-PROCESS-NEXT: [[TMP202:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP203:%.*]] = inttoptr i32 [[TMP202]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP204:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP203]], i32 -30 +; POST-PROCESS-NEXT: [[TMP205:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP204]], i32 0, i32 0, i64 36 +; POST-PROCESS-NEXT: store i32 [[TMP77]], ptr addrspace(21) [[TMP205]], align 4 +; POST-PROCESS-NEXT: [[TMP206:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP207:%.*]] = inttoptr i32 [[TMP206]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP208:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP207]], i32 -30 +; POST-PROCESS-NEXT: [[TMP209:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP208]], i32 0, i32 0, i64 37 +; POST-PROCESS-NEXT: store i32 [[TMP82]], ptr addrspace(21) [[TMP209]], align 4 +; POST-PROCESS-NEXT: [[TMP210:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP211:%.*]] = inttoptr i32 [[TMP210]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP212:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP211]], i32 -30 +; POST-PROCESS-NEXT: [[TMP213:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP212]], i32 0, i32 0, i64 38 +; POST-PROCESS-NEXT: store i32 [[TMP87]], ptr addrspace(21) [[TMP213]], align 4 +; POST-PROCESS-NEXT: [[TMP214:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP215:%.*]] = inttoptr i32 [[TMP214]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP216:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP215]], i32 -30 +; POST-PROCESS-NEXT: [[TMP217:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP216]], i32 0, i32 0, i64 39 +; POST-PROCESS-NEXT: store i32 [[TMP92]], ptr addrspace(21) [[TMP217]], align 4 +; POST-PROCESS-NEXT: [[TMP218:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP219:%.*]] = inttoptr i32 [[TMP218]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP220:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP219]], i32 -30 +; POST-PROCESS-NEXT: [[TMP221:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP220]], i32 0, i32 0, i64 40 +; POST-PROCESS-NEXT: store i32 [[TMP97]], ptr addrspace(21) [[TMP221]], align 4 +; POST-PROCESS-NEXT: [[TMP222:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP223:%.*]] = inttoptr i32 [[TMP222]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP224:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP223]], i32 -30 +; POST-PROCESS-NEXT: [[TMP225:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP224]], i32 0, i32 0, i64 41 +; POST-PROCESS-NEXT: store i32 [[TMP102]], ptr addrspace(21) [[TMP225]], align 4 +; POST-PROCESS-NEXT: [[TMP226:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP227:%.*]] = inttoptr i32 [[TMP226]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP228:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP227]], i32 -30 +; POST-PROCESS-NEXT: [[TMP229:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP228]], i32 0, i32 0, i64 42 +; POST-PROCESS-NEXT: store i32 [[TMP107]], ptr addrspace(21) [[TMP229]], align 4 +; POST-PROCESS-NEXT: [[TMP230:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP231:%.*]] = inttoptr i32 [[TMP230]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP232:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP231]], i32 -30 +; POST-PROCESS-NEXT: [[TMP233:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP232]], i32 0, i32 0, i64 43 +; POST-PROCESS-NEXT: store i32 [[TMP112]], ptr addrspace(21) [[TMP233]], align 4 +; POST-PROCESS-NEXT: [[TMP234:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP235:%.*]] = inttoptr i32 [[TMP234]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP236:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP235]], i32 -30 +; POST-PROCESS-NEXT: [[TMP237:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP236]], i32 0, i32 0, i64 44 +; POST-PROCESS-NEXT: store i32 [[TMP117]], ptr addrspace(21) [[TMP237]], align 4 +; POST-PROCESS-NEXT: [[TMP238:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP239:%.*]] = inttoptr i32 [[TMP238]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP240:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP239]], i32 -30 +; POST-PROCESS-NEXT: [[TMP241:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP240]], i32 0, i32 0, i64 45 +; POST-PROCESS-NEXT: store i32 [[TMP122]], ptr addrspace(21) [[TMP241]], align 4 +; POST-PROCESS-NEXT: [[TMP242:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP243:%.*]] = inttoptr i32 [[TMP242]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP244:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP243]], i32 -30 +; POST-PROCESS-NEXT: [[TMP245:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP244]], i32 0, i32 0, i64 46 +; POST-PROCESS-NEXT: store i32 [[TMP127]], ptr addrspace(21) [[TMP245]], align 4 +; POST-PROCESS-NEXT: [[TMP246:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP247:%.*]] = inttoptr i32 [[TMP246]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP248:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP247]], i32 -30 +; POST-PROCESS-NEXT: [[TMP249:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP248]], i32 0, i32 0, i64 47 +; POST-PROCESS-NEXT: store i32 [[TMP132]], ptr addrspace(21) [[TMP249]], align 4 +; POST-PROCESS-NEXT: [[TMP250:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP251:%.*]] = inttoptr i32 [[TMP250]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP252:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP251]], i32 -30 +; POST-PROCESS-NEXT: [[TMP253:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP252]], i32 0, i32 0, i64 48 +; POST-PROCESS-NEXT: store i32 [[TMP137]], ptr addrspace(21) [[TMP253]], align 4 +; POST-PROCESS-NEXT: [[TMP254:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP255:%.*]] = inttoptr i32 [[TMP254]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP256:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP255]], i32 -30 +; POST-PROCESS-NEXT: [[TMP257:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP256]], i32 0, i32 0, i64 49 +; POST-PROCESS-NEXT: store i32 [[TMP142]], ptr addrspace(21) [[TMP257]], align 4 +; POST-PROCESS-NEXT: [[TMP258:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP259:%.*]] = inttoptr i32 [[TMP258]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP260:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP259]], i32 -30 +; POST-PROCESS-NEXT: [[TMP261:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP260]], i32 0, i32 0, i64 50 +; POST-PROCESS-NEXT: store i32 [[TMP147]], ptr addrspace(21) [[TMP261]], align 4 +; POST-PROCESS-NEXT: [[TMP262:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP263:%.*]] = inttoptr i32 [[TMP262]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP264:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP263]], i32 -30 +; POST-PROCESS-NEXT: [[TMP265:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP264]], i32 0, i32 0, i64 51 +; POST-PROCESS-NEXT: store i32 [[TMP152]], ptr addrspace(21) [[TMP265]], align 4 +; POST-PROCESS-NEXT: [[TMP266:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP267:%.*]] = inttoptr i32 [[TMP266]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP268:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP267]], i32 -30 +; POST-PROCESS-NEXT: [[TMP269:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP268]], i32 0, i32 0, i64 52 +; POST-PROCESS-NEXT: store i32 [[TMP157]], ptr addrspace(21) [[TMP269]], align 4 +; POST-PROCESS-NEXT: [[TMP270:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP271:%.*]] = inttoptr i32 [[TMP270]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP272:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP271]], i32 -30 +; POST-PROCESS-NEXT: [[TMP273:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP272]], i32 0, i32 0, i64 53 +; POST-PROCESS-NEXT: store i32 [[TMP162]], ptr addrspace(21) [[TMP273]], align 4 +; POST-PROCESS-NEXT: [[TMP274:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP275:%.*]] = inttoptr i32 [[TMP274]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP276:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP275]], i32 -30 +; POST-PROCESS-NEXT: [[TMP277:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP276]], i32 0, i32 0, i64 54 +; POST-PROCESS-NEXT: store i32 [[TMP167]], ptr addrspace(21) [[TMP277]], align 4 +; POST-PROCESS-NEXT: [[TMP278:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP279:%.*]] = inttoptr i32 [[TMP278]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP280:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP279]], i32 -30 +; POST-PROCESS-NEXT: [[TMP281:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP280]], i32 0, i32 0, i64 55 +; POST-PROCESS-NEXT: store i32 [[TMP172]], ptr addrspace(21) [[TMP281]], align 4 +; POST-PROCESS-NEXT: [[TMP282:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; POST-PROCESS-NEXT: [[TMP283:%.*]] = inttoptr i32 [[TMP282]] to ptr addrspace(21) +; POST-PROCESS-NEXT: [[TMP284:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP283]], i32 -30 +; POST-PROCESS-NEXT: [[TMP285:%.*]] = getelementptr [[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspace(21) [[TMP284]], i32 0, i32 0, i64 56 +; POST-PROCESS-NEXT: store i32 [[TMP177]], ptr addrspace(21) [[TMP285]], align 4 +; POST-PROCESS-NEXT: [[TMP286:%.*]] = load i32, ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP287:%.*]] = add i32 [[TMP286]], -108 +; POST-PROCESS-NEXT: store i32 [[TMP287]], ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: [[TMP288:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; POST-PROCESS-NEXT: [[DOTFCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[TMP288]], i32 0, i32 0 ; POST-PROCESS-NEXT: [[DOTFCA_0_LOAD:%.*]] = load i32, ptr [[DOTFCA_0_GEP]], align 4 ; POST-PROCESS-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, i32 [[DOTFCA_0_LOAD]], 0 -; POST-PROCESS-NEXT: [[TMP301:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP302:%.*]] = add i32 [[TMP301]], 0 -; POST-PROCESS-NEXT: store i32 [[TMP302]], ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: [[TMP303:%.*]] = inttoptr i32 [[TMP302]] to ptr addrspace(21) -; POST-PROCESS-NEXT: [[TMP304:%.*]] = load i32, ptr [[CSP]], align 4 -; POST-PROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP304]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount !17 +; POST-PROCESS-NEXT: [[TMP289:%.*]] = load i32, ptr [[CSP]], align 4 +; POST-PROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP289]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount !17 ; POST-PROCESS-NEXT: unreachable ; diff --git a/shared/continuations/test/dx/register-buffer.ll b/shared/continuations/test/dx/register-buffer.ll index 833150a9a2..ca58edbd1d 100644 --- a/shared/continuations/test/dx/register-buffer.ll +++ b/shared/continuations/test/dx/register-buffer.ll @@ -1,18 +1,23 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt --verify-each -passes='register-buffer,lint,instsimplify' -S %s 2>%t.stderr | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --version 3 +; RUN: opt --verify-each -passes='register-buffer,lint,instsimplify' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" -; CHECK: @GLOBAL = external addrspace(20) global [15 x i32] @GLOBAL = external global [20 x i32], !registerbuffer !1 +@GLOBAL_NO_REGS = external global [20 x i32], !registerbuffer !2 !1 = !{ i32 15, i32 21 } +!2 = !{ i32 0, i32 21 } %complex_type = type { %complex_type*, half, %complex_type addrspace(1)* } declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) +;. +; CHECK: @[[GLOBAL:[a-zA-Z0-9_$"\\.-]+]] = external addrspace(20) global [15 x i32] +; CHECK: @[[GLOBAL_NO_REGS:[a-zA-Z0-9_$"\\.-]+]] = external addrspace(20) global [0 x i32] +;. define i32 @load_i32_reg() { ; CHECK-LABEL: define i32 @load_i32_reg() { ; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr addrspace(20) addrspacecast (ptr getelementptr (i8, ptr addrspacecast (ptr addrspace(20) @GLOBAL to ptr), i64 40) to ptr addrspace(20)), align 4 @@ -73,6 +78,19 @@ define i32 @load_i32_dyn(i32 %i) { ret i32 %val } +define i32 @load_i32_dyn_no_regs(i32 %i) { +; CHECK-LABEL: define i32 @load_i32_dyn_no_regs( +; CHECK-SAME: i32 [[I:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = call ptr addrspace(21) @registerbuffer.getpointer.a0i32(ptr addrspace(20) @GLOBAL_NO_REGS) +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr [20 x i32], ptr addrspace(21) [[TMP1]], i32 0, i32 [[I]] +; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr addrspace(21) [[TMP2]], align 4 +; CHECK-NEXT: ret i32 [[TMP3]] +; + %addr = getelementptr [20 x i32], [20 x i32]* @GLOBAL_NO_REGS, i32 0, i32 %i + %val = load i32, i32* %addr + ret i32 %val +} + define i64 @load_i64_reg() { ; CHECK-LABEL: define i64 @load_i64_reg() { ; CHECK-NEXT: [[VAL_FCA_ALLOCA:%.*]] = alloca <{ i32, i32 }>, align 8 @@ -933,3 +951,7 @@ define void @with_lifetime_intrinsics() { call void @llvm.lifetime.end.p0i8(i64 80, i8* %ptr) ret void } +;. +; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } +; CHECK: attributes #[[ATTR1:[0-9]+]] = { nofree norecurse nosync nounwind willreturn memory(read) } +;. diff --git a/shared/continuations/test/dx/remat-intrinsic.ll b/shared/continuations/test/dx/remat-intrinsic.ll index 0f027bacf9..8073d3b6d8 100644 --- a/shared/continuations/test/dx/remat-intrinsic.ll +++ b/shared/continuations/test/dx/remat-intrinsic.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 3 -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' \ -; RUN: -S %s 2>%t.stderr | FileCheck -check-prefix=SAVESTATE %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' \ +; RUN: -S %s 2> %t.stderr | FileCheck -check-prefix=SAVESTATE %s ; RUN: count 0 < %t.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" @@ -131,49 +131,39 @@ attributes #1 = { nounwind } ; SAVESTATE-NEXT: [[CSP:%.*]] = alloca i32, align 4 ; SAVESTATE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP0]], ptr [[SYSTEM_DATA]], align 4 ; SAVESTATE-NEXT: store i32 [[CSPINIT]], ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP1:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 0 -; SAVESTATE-NEXT: store i32 [[TMP2]], ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP1]] to ptr addrspace(21) ; SAVESTATE-NEXT: [[RETURNADDR_SPILL_ADDR:%.*]] = getelementptr inbounds [[CALLED_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; SAVESTATE-NEXT: store i64 [[RETURNADDR]], ptr [[RETURNADDR_SPILL_ADDR]], align 4 -; SAVESTATE-NEXT: [[TMP4:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA]], align 4 -; SAVESTATE-NEXT: [[DOTFCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP4]], 0 +; SAVESTATE-NEXT: [[TMP1:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA]], align 4 +; SAVESTATE-NEXT: [[DOTFCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], 0 ; SAVESTATE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) -; SAVESTATE-NEXT: [[TMP5:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 -; SAVESTATE-NEXT: [[TMP6:%.*]] = call <3 x i32> @_cont_DispatchRaysIndex3(ptr [[SYSTEM_DATA]]) -; SAVESTATE-NEXT: [[I:%.*]] = extractelement <3 x i32> [[TMP6]], i8 0 +; SAVESTATE-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; SAVESTATE-NEXT: [[TMP3:%.*]] = call <3 x i32> @_cont_DispatchRaysIndex3(ptr [[SYSTEM_DATA]]) +; SAVESTATE-NEXT: [[I:%.*]] = extractelement <3 x i32> [[TMP3]], i8 0 ; SAVESTATE-NEXT: [[UNPACKED:%.*]] = call [[DX_TYPES_FOURI32:%.*]] @dx.op.unpack4x8.i32(i32 219, i8 1, i32 [[I]]) ; SAVESTATE-NEXT: [[HANDLE0:%.*]] = load [[DX_TYPES_HANDLE:%.*]], ptr @"\01?RenderTarget@@3V?$RWTexture2D@V?$vector@M$03@@@@A", align 4 ; SAVESTATE-NEXT: [[HANDLE1:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[HANDLE0]]) ; SAVESTATE-NEXT: [[HANDLE2:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[HANDLE1]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) ; SAVESTATE-NEXT: [[DIS_DATA_I_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, i32 [[DOTFCA_0_EXTRACT]], 0 -; SAVESTATE-NEXT: store i32 [[TMP5]], ptr addrspace(20) @REGISTERS, align 4 -; SAVESTATE-NEXT: [[TMP7:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], 8 -; SAVESTATE-NEXT: store i32 [[TMP8]], ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; SAVESTATE-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4 -; SAVESTATE-NEXT: [[TMP11:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP12:%.*]] = inttoptr i32 [[TMP11]] to ptr addrspace(21) -; SAVESTATE-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP12]], i64 -8 -; SAVESTATE-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP13]], i32 0 -; SAVESTATE-NEXT: store i32 [[TMP10]], ptr addrspace(21) [[TMP14]], align 4 -; SAVESTATE-NEXT: [[TMP15:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; SAVESTATE-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4 -; SAVESTATE-NEXT: [[TMP17:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP18:%.*]] = inttoptr i32 [[TMP17]] to ptr addrspace(21) -; SAVESTATE-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP18]], i64 -8 -; SAVESTATE-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP19]], i32 0 -; SAVESTATE-NEXT: [[TMP21:%.*]] = getelementptr [2 x i32], ptr addrspace(21) [[TMP20]], i32 0, i32 1 -; SAVESTATE-NEXT: store i32 [[TMP16]], ptr addrspace(21) [[TMP21]], align 4 -; SAVESTATE-NEXT: [[TMP22:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP23:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @called.resume.0 to i64)) -; SAVESTATE-NEXT: call void (i64, ...) @continuation.continue(i64 2, i32 [[TMP22]], i64 [[TMP23]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I_FCA_0_INSERT]]), !continuation.registercount !17, !continuation.returnedRegistercount !17 +; SAVESTATE-NEXT: store i32 [[TMP2]], ptr addrspace(20) @REGISTERS, align 4 +; SAVESTATE-NEXT: [[TMP4:%.*]] = load i32, ptr [[CSP]], align 4 +; SAVESTATE-NEXT: [[TMP5:%.*]] = add i32 [[TMP4]], 8 +; SAVESTATE-NEXT: store i32 [[TMP5]], ptr [[CSP]], align 4 +; SAVESTATE-NEXT: [[TMP6:%.*]] = inttoptr i32 [[TMP4]] to ptr addrspace(21) +; SAVESTATE-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; SAVESTATE-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP6]], i32 0, i32 0 +; SAVESTATE-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4 +; SAVESTATE-NEXT: store i32 [[TMP9]], ptr addrspace(21) [[TMP8]], align 4 +; SAVESTATE-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; SAVESTATE-NEXT: [[TMP11:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP6]], i32 0, i32 1 +; SAVESTATE-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP10]], align 4 +; SAVESTATE-NEXT: store i32 [[TMP12]], ptr addrspace(21) [[TMP11]], align 4 +; SAVESTATE-NEXT: [[TMP13:%.*]] = load i32, ptr [[CSP]], align 4 +; SAVESTATE-NEXT: [[TMP14:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @called.resume.0 to i64)) +; SAVESTATE-NEXT: call void (i64, ...) @continuation.continue(i64 2, i32 [[TMP13]], i64 [[TMP14]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I_FCA_0_INSERT]]), !continuation.registercount !17, !continuation.returnedRegistercount !17 ; SAVESTATE-NEXT: unreachable ; ; -; SAVESTATE-LABEL: define void @called.resume.0( +; SAVESTATE-LABEL: define dso_local void @called.resume.0( ; SAVESTATE-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !continuation !15 !lgc.rt.shaderstage !16 !continuation.registercount !17 { ; SAVESTATE-NEXT: entryresume.0: ; SAVESTATE-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA]], align 8 @@ -181,25 +171,21 @@ attributes #1 = { nounwind } ; SAVESTATE-NEXT: [[CSP:%.*]] = alloca i32, align 4 ; SAVESTATE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], ptr [[SYSTEM_DATA]], align 4 ; SAVESTATE-NEXT: store i32 [[TMP0]], ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP2:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; SAVESTATE-NEXT: [[TMP3:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP4:%.*]] = inttoptr i32 [[TMP3]] to ptr addrspace(21) -; SAVESTATE-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP4]], i64 -8 -; SAVESTATE-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP5]], i32 0 -; SAVESTATE-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(21) [[TMP6]], align 4 -; SAVESTATE-NEXT: store i32 [[TMP7]], ptr [[TMP2]], align 4 -; SAVESTATE-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; SAVESTATE-NEXT: [[TMP9:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP10:%.*]] = inttoptr i32 [[TMP9]] to ptr addrspace(21) -; SAVESTATE-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP10]], i64 -8 -; SAVESTATE-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP11]], i32 0 -; SAVESTATE-NEXT: [[TMP13:%.*]] = getelementptr [2 x i32], ptr addrspace(21) [[TMP12]], i32 0, i32 1 -; SAVESTATE-NEXT: [[TMP14:%.*]] = load i32, ptr addrspace(21) [[TMP13]], align 4 -; SAVESTATE-NEXT: store i32 [[TMP14]], ptr [[TMP8]], align 4 -; SAVESTATE-NEXT: [[TMP15:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], -8 -; SAVESTATE-NEXT: store i32 [[TMP16]], ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP17:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 +; SAVESTATE-NEXT: [[TMP2:%.*]] = load i32, ptr [[CSP]], align 4 +; SAVESTATE-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(21) +; SAVESTATE-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP3]], i64 -8 +; SAVESTATE-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 0 +; SAVESTATE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; SAVESTATE-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(21) [[TMP5]], align 4 +; SAVESTATE-NEXT: store i32 [[TMP7]], ptr [[TMP6]], align 4 +; SAVESTATE-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 1 +; SAVESTATE-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; SAVESTATE-NEXT: [[TMP10:%.*]] = load i32, ptr addrspace(21) [[TMP8]], align 4 +; SAVESTATE-NEXT: store i32 [[TMP10]], ptr [[TMP9]], align 4 +; SAVESTATE-NEXT: [[TMP11:%.*]] = load i32, ptr [[CSP]], align 4 +; SAVESTATE-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], -8 +; SAVESTATE-NEXT: store i32 [[TMP12]], ptr [[CSP]], align 4 +; SAVESTATE-NEXT: [[TMP13:%.*]] = load i32, ptr addrspace(20) @REGISTERS, align 4 ; SAVESTATE-NEXT: [[DOTFCA_0_EXTRACT3:%.*]] = extractvalue [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], 0 ; SAVESTATE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; SAVESTATE-NEXT: [[RETURNADDR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[CALLED_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 @@ -207,17 +193,17 @@ attributes #1 = { nounwind } ; SAVESTATE-NEXT: [[HANDLE011:%.*]] = load [[DX_TYPES_HANDLE:%.*]], ptr @"\01?RenderTarget@@3V?$RWTexture2D@V?$vector@M$03@@@@A", align 4 ; SAVESTATE-NEXT: [[HANDLE110:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[HANDLE011]]) ; SAVESTATE-NEXT: [[HANDLE29:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[HANDLE110]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) -; SAVESTATE-NEXT: [[TMP18:%.*]] = call <3 x i32> @_cont_DispatchRaysIndex3(ptr [[SYSTEM_DATA]]) -; SAVESTATE-NEXT: [[I8:%.*]] = extractelement <3 x i32> [[TMP18]], i8 0 +; SAVESTATE-NEXT: [[TMP14:%.*]] = call <3 x i32> @_cont_DispatchRaysIndex3(ptr [[SYSTEM_DATA]]) +; SAVESTATE-NEXT: [[I8:%.*]] = extractelement <3 x i32> [[TMP14]], i8 0 ; SAVESTATE-NEXT: [[UNPACKED7:%.*]] = call [[DX_TYPES_FOURI32:%.*]] @dx.op.unpack4x8.i32(i32 219, i8 1, i32 [[I8]]) -; SAVESTATE-NEXT: [[TMP19:%.*]] = call <3 x i32> @_cont_DispatchRaysIndex3(ptr [[SYSTEM_DATA]]) -; SAVESTATE-NEXT: [[I6:%.*]] = extractelement <3 x i32> [[TMP19]], i8 0 +; SAVESTATE-NEXT: [[TMP15:%.*]] = call <3 x i32> @_cont_DispatchRaysIndex3(ptr [[SYSTEM_DATA]]) +; SAVESTATE-NEXT: [[I6:%.*]] = extractelement <3 x i32> [[TMP15]], i8 0 ; SAVESTATE-NEXT: [[UNPACKED5:%.*]] = call [[DX_TYPES_FOURI32]] @dx.op.unpack4x8.i32(i32 219, i8 1, i32 [[I6]]) -; SAVESTATE-NEXT: [[TMP20:%.*]] = call <3 x i32> @_cont_DispatchRaysIndex3(ptr [[SYSTEM_DATA]]) -; SAVESTATE-NEXT: [[I4:%.*]] = extractelement <3 x i32> [[TMP20]], i8 0 +; SAVESTATE-NEXT: [[TMP16:%.*]] = call <3 x i32> @_cont_DispatchRaysIndex3(ptr [[SYSTEM_DATA]]) +; SAVESTATE-NEXT: [[I4:%.*]] = extractelement <3 x i32> [[TMP16]], i8 0 ; SAVESTATE-NEXT: [[UNPACKED3:%.*]] = call [[DX_TYPES_FOURI32]] @dx.op.unpack4x8.i32(i32 219, i8 1, i32 [[I4]]) -; SAVESTATE-NEXT: [[TMP21:%.*]] = call <3 x i32> @_cont_DispatchRaysIndex3(ptr [[SYSTEM_DATA]]) -; SAVESTATE-NEXT: [[I2:%.*]] = extractelement <3 x i32> [[TMP21]], i8 0 +; SAVESTATE-NEXT: [[TMP17:%.*]] = call <3 x i32> @_cont_DispatchRaysIndex3(ptr [[SYSTEM_DATA]]) +; SAVESTATE-NEXT: [[I2:%.*]] = extractelement <3 x i32> [[TMP17]], i8 0 ; SAVESTATE-NEXT: [[UNPACKED1:%.*]] = call [[DX_TYPES_FOURI32]] @dx.op.unpack4x8.i32(i32 219, i8 1, i32 [[I2]]) ; SAVESTATE-NEXT: [[A:%.*]] = extractvalue [[DX_TYPES_FOURI32]] [[UNPACKED7]], 0 ; SAVESTATE-NEXT: [[B:%.*]] = extractvalue [[DX_TYPES_FOURI32]] [[UNPACKED5]], 1 @@ -227,11 +213,7 @@ attributes #1 = { nounwind } ; SAVESTATE-NEXT: call void @dx.op.textureStore.f32(i32 67, [[DX_TYPES_HANDLE]] [[HANDLE29]], i32 0, i32 0, i32 undef, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 1.000000e+00, i8 15) ; SAVESTATE-NEXT: store i32 [[PACKED]], ptr addrspace(20) @REGISTERS, align 4 ; SAVESTATE-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA]] poison, i32 [[DOTFCA_0_EXTRACT3]], 0 -; SAVESTATE-NEXT: [[TMP22:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP23:%.*]] = add i32 [[TMP22]], 0 -; SAVESTATE-NEXT: store i32 [[TMP23]], ptr [[CSP]], align 4 -; SAVESTATE-NEXT: [[TMP24:%.*]] = inttoptr i32 [[TMP23]] to ptr addrspace(21) -; SAVESTATE-NEXT: [[TMP25:%.*]] = load i32, ptr [[CSP]], align 4 -; SAVESTATE-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP25]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount !17 +; SAVESTATE-NEXT: [[TMP18:%.*]] = load i32, ptr [[CSP]], align 4 +; SAVESTATE-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP18]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount !17 ; SAVESTATE-NEXT: unreachable ; diff --git a/shared/continuations/test/dx/remove-types-metadata.ll b/shared/continuations/test/dx/remove-types-metadata.ll index 81ca8010db..c46a10a41d 100644 --- a/shared/continuations/test/dx/remove-types-metadata.ll +++ b/shared/continuations/test/dx/remove-types-metadata.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-attributes --check-globals --version 2 -; RUN: opt --verify-each -passes='remove-types-metadata' -S %s 2>%t.stderr | FileCheck -check-prefix=METADATA %s +; RUN: opt --verify-each -passes='remove-types-metadata' -S %s 2> %t.stderr | FileCheck -check-prefix=METADATA %s ; RUN: count 0 < %t.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" diff --git a/shared/continuations/test/dx/remove-unused-declarations.ll b/shared/continuations/test/dx/remove-unused-declarations.ll index dfe9a1f116..0ad97a9fdb 100644 --- a/shared/continuations/test/dx/remove-unused-declarations.ll +++ b/shared/continuations/test/dx/remove-unused-declarations.ll @@ -1,6 +1,6 @@ -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint' -S %s 2>%t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE-DECL %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint' -S %s 2> %t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE-DECL %s ; RUN: count 0 < %t0.stderr -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint' -S %s 2>%t1.stderr | FileCheck -check-prefix=DXILCONTPOSTPROCESS-DECL %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint,dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint' -S %s 2> %t1.stderr | FileCheck -check-prefix=DXILCONTPOSTPROCESS-DECL %s ; RUN: count 0 < %t1.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" diff --git a/shared/continuations/test/dx/save-continuation-state.ll b/shared/continuations/test/dx/save-continuation-state.ll deleted file mode 100644 index 917689b0de..0000000000 --- a/shared/continuations/test/dx/save-continuation-state.ll +++ /dev/null @@ -1,127 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3 -; RUN: opt --verify-each -passes='save-continuation-state,lint' -S %s 2>%t.stderr | FileCheck %s -; RUN: count 0 < %t.stderr - -target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" - -%continuation.token = type {} -%simple_await.Frame = type { i64 } - -@RETURN_ADDR = external addrspace(20) global i64 -@CONTINUATION_STATE = external addrspace(20) global [0 x i32] - -declare %continuation.token* @async_fun() - -declare void @continuation.save.continuation_state() - -declare void @continuation.restore.continuation_state() - -declare void @continuation.continue(i64, ...) - -define void @simple_await(i32 %0, i64 %1) !continuation !1 !continuation.state !2 { -; CHECK-LABEL: define void @simple_await( -; CHECK-SAME: i32 [[TMP0:%.*]], i64 [[TMP1:%.*]]) !continuation !1 !continuation.state !2 !continuation.stacksize !3 { -; CHECK-NEXT: [[CSP:%.*]] = alloca i32, align 4 -; CHECK-NEXT: store i32 [[TMP0]], ptr [[CSP]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[CSP]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 0 -; CHECK-NEXT: store i32 [[TMP4]], ptr [[CSP]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = inttoptr i32 [[TMP3]] to ptr addrspace(21) -; CHECK-NEXT: [[FRAMEPTR:%.*]] = bitcast ptr addrspacecast (ptr addrspace(20) @CONTINUATION_STATE to ptr) to ptr -; CHECK-NEXT: [[RET_ADDR:%.*]] = load i64, ptr addrspace(20) @RETURN_ADDR, align 8 -; CHECK-NEXT: [[DOTSPILL_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_FRAME:%.*]], ptr [[FRAMEPTR]], i32 0, i32 0 -; CHECK-NEXT: store i64 [[RET_ADDR]], ptr [[DOTSPILL_ADDR]], align 8 -; CHECK-NEXT: [[RESUME_FUNC:%.*]] = bitcast ptr @simple_await.resume.0 to ptr -; CHECK-NEXT: [[TMP6:%.*]] = insertvalue { ptr, ptr } undef, ptr [[RESUME_FUNC]], 0 -; CHECK-NEXT: [[TMP7:%.*]] = ptrtoint ptr [[RESUME_FUNC]] to i64 -; CHECK-NEXT: store i64 [[TMP7]], ptr addrspace(20) @RETURN_ADDR, align 8 -; CHECK-NEXT: call void (i64, ...) @continuation.continue(i64 ptrtoint (ptr @async_fun to i64)) -; CHECK-NEXT: unreachable -; - call void @continuation.save.continuation_state() - %FramePtr = bitcast i8* addrspacecast (i8 addrspace(20)* bitcast ([0 x i32] addrspace(20)* @CONTINUATION_STATE to i8 addrspace(20)*) to i8*) to %simple_await.Frame* - %ret_addr = load i64, i64 addrspace(20)* @RETURN_ADDR, align 8 - %.spill.addr = getelementptr inbounds %simple_await.Frame, %simple_await.Frame* %FramePtr, i32 0, i32 0 - store i64 %ret_addr, i64* %.spill.addr, align 8 - %resume_func = bitcast { i8*, %continuation.token* } (i8*, i1)* bitcast (void (i32)* @simple_await.resume.0 to { i8*, %continuation.token* } (i8*, i1)*) to i8* - %3 = insertvalue { i8*, %continuation.token* } undef, i8* %resume_func, 0 - %4 = ptrtoint i8* %resume_func to i64 - store i64 %4, i64 addrspace(20)* @RETURN_ADDR, align 8 - call void (i64, ...) @continuation.continue(i64 ptrtoint (%continuation.token* ()* @async_fun to i64)) - unreachable -} - -define internal void @simple_await.resume.0(i32 %0) !continuation !1 { -; CHECK-LABEL: define internal void @simple_await.resume.0( -; CHECK-SAME: i32 [[TMP0:%.*]]) !continuation !1 { -; CHECK-NEXT: [[CSP:%.*]] = alloca i32, align 4 -; CHECK-NEXT: store i32 [[TMP0]], ptr [[CSP]], align 4 -; CHECK-NEXT: [[FRAMEPTR:%.*]] = bitcast ptr addrspacecast (ptr addrspace(20) @CONTINUATION_STATE to ptr) to ptr -; CHECK-NEXT: [[VFRAME:%.*]] = bitcast ptr [[FRAMEPTR]] to ptr -; CHECK-NEXT: [[DOTRELOAD_ADDR:%.*]] = getelementptr inbounds [[SIMPLE_AWAIT_FRAME:%.*]], ptr [[FRAMEPTR]], i32 0, i32 0 -; CHECK-NEXT: [[DOTRELOAD:%.*]] = load i64, ptr [[DOTRELOAD_ADDR]], align 8 -; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[CSP]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[TMP2]], 0 -; CHECK-NEXT: store i32 [[TMP3]], ptr [[CSP]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = inttoptr i32 [[TMP3]] to ptr addrspace(21) -; CHECK-NEXT: call void (i64, ...) @continuation.continue(i64 [[DOTRELOAD]]) -; CHECK-NEXT: unreachable -; - %FramePtr = bitcast i8* addrspacecast (i8 addrspace(20)* bitcast ([0 x i32] addrspace(20)* @CONTINUATION_STATE to i8 addrspace(20)*) to i8*) to %simple_await.Frame* - %vFrame = bitcast %simple_await.Frame* %FramePtr to i8* - %.reload.addr = getelementptr inbounds %simple_await.Frame, %simple_await.Frame* %FramePtr, i32 0, i32 0 - %.reload = load i64, i64* %.reload.addr, align 8 - call void @continuation.restore.continuation_state() - call void (i64, ...) @continuation.continue(i64 %.reload) - unreachable -} - -define void @odd_cont_state_size(i32 %0, i64 %1) !continuation !3 !continuation.state !4 { -; CHECK-LABEL: define void @odd_cont_state_size( -; CHECK-SAME: i32 [[TMP0:%.*]], i64 [[TMP1:%.*]]) !continuation !4 !continuation.state !5 !continuation.stacksize !3 { -; CHECK-NEXT: [[CSP:%.*]] = alloca i32, align 4 -; CHECK-NEXT: store i32 [[TMP0]], ptr [[CSP]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[CSP]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 0 -; CHECK-NEXT: store i32 [[TMP4]], ptr [[CSP]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = inttoptr i32 [[TMP3]] to ptr addrspace(21) -; CHECK-NEXT: call void (i64, ...) @continuation.continue(i64 2) -; CHECK-NEXT: unreachable -; - call void @continuation.save.continuation_state() - call void (i64, ...) @continuation.continue(i64 2) - unreachable -} - -; Continuation state is larger than the register size -define void @large_cont_state_size(i32 %0, i64 %1) !continuation !5 !continuation.state !6 { -; CHECK-LABEL: define void @large_cont_state_size( -; CHECK-SAME: i32 [[TMP0:%.*]], i64 [[TMP1:%.*]]) !continuation !6 !continuation.state !7 !continuation.stacksize !3 { -; CHECK-NEXT: [[CSP:%.*]] = alloca i32, align 4 -; CHECK-NEXT: store i32 [[TMP0]], ptr [[CSP]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[CSP]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 0 -; CHECK-NEXT: store i32 [[TMP4]], ptr [[CSP]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = inttoptr i32 [[TMP3]] to ptr addrspace(21) -; CHECK-NEXT: call void (i64, ...) @continuation.continue(i64 2) -; CHECK-NEXT: unreachable -; - call void @continuation.save.continuation_state() - call void (i64, ...) @continuation.continue(i64 2) - unreachable -} - -attributes #0 = { nounwind } - -!continuation.stackAddrspace = !{!0} - -!0 = !{i32 21} - -!1 = !{void (i32, i64)* @simple_await} -!2 = !{i32 8} - -!3 = !{void (i32, i64)* @odd_cont_state_size} -!4 = !{i32 2} - -!5 = !{void (i32, i64)* @large_cont_state_size} -!6 = !{i32 2048} diff --git a/shared/continuations/test/dx/traceray.ll b/shared/continuations/test/dx/traceray.ll index 98f262bce3..b45c2701b3 100644 --- a/shared/continuations/test/dx/traceray.ll +++ b/shared/continuations/test/dx/traceray.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 3 -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2>%t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2> %t0.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s ; RUN: count 0 < %t0.stderr -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,inline,lint,pre-coroutine-lowering,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2>%t1.stderr | FileCheck -check-prefix=DXILCONTPOSTPROCESS %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,inline,lint,lower-raytracing-pipeline,lint,sroa,lint,lower-await,lint,coro-early,dxil-coro-split,coro-cleanup,lint,legacy-cleanup-continuations,lint,register-buffer,lint,save-continuation-state,lint,dxil-cont-post-process,lint,remove-types-metadata' -S %s 2> %t1.stderr | FileCheck -check-prefix=DXILCONTPOSTPROCESS %s ; RUN: count 0 < %t1.stderr target datalayout = "e-m:e-p:64:32-p20:32:32-p21:32:32-i1:32-i8:8-i16:32-i32:32-i64:32-f16:32-f32:32-f64:32-v16:32-v32:32-v48:32-v64:32-v80:32-v96:32-v112:32-v128:32-v144:32-v160:32-v176:32-v192:32-v208:32-v224:32-v240:32-v256:32-n8:16:32" @@ -393,174 +393,6 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; LOWERRAYTRACINGPIPELINE-NEXT: ret i32 5 ; ; -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_TraceRay.struct.RayPayload.attr_max_32_bytes( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i64 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]], i32 [[TMP3:%.*]], i32 [[TMP4:%.*]], i32 [[TMP5:%.*]], float [[TMP6:%.*]], float [[TMP7:%.*]], float [[TMP8:%.*]], float [[TMP9:%.*]], float [[TMP10:%.*]], float [[TMP11:%.*]], float [[TMP12:%.*]], float [[TMP13:%.*]], ptr [[TMP14:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[ADDR:%.*]] = call i64 @_AmdGetResumePointAddr() #[[ATTR3:[0-9]+]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA2:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA]], i64 [[ADDR]], 5 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD:%.*]], ptr [[TMP14]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = getelementptr i32, ptr [[TMP16]], i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[TMP17]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP18]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP19]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[TMP16]], i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[TMP20]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP22]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP23:%.*]] = getelementptr i32, ptr [[TMP20]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP24]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr [[TMP20]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP25]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP26]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP27:%.*]] = call ptr inttoptr (i64 4 to ptr)(i64 -1, [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA2]]), !continuation.registercount !35, !continuation.returnedRegistercount !35, !continuation.wait.await !14 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @await.struct.DispatchSystemData(ptr [[TMP27]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_RAYPAYLOAD]] poison, ptr [[TMP14]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP14]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP30:%.*]] = getelementptr i32, ptr [[TMP29]], i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP31:%.*]] = getelementptr i32, ptr [[TMP30]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP32:%.*]] = load i32, ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP32]], ptr [[TMP31]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP33:%.*]] = getelementptr i32, ptr [[TMP29]], i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = getelementptr i32, ptr [[TMP33]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP35:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP35]], ptr [[TMP34]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP36:%.*]] = getelementptr i32, ptr [[TMP33]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP37:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP37]], ptr [[TMP36]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = getelementptr i32, ptr [[TMP33]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP39]], ptr [[TMP38]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP28]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_TraceRay( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i64 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]], i32 [[TMP3:%.*]], i32 [[TMP4:%.*]], i32 [[TMP5:%.*]], float [[TMP6:%.*]], float [[TMP7:%.*]], float [[TMP8:%.*]], float [[TMP9:%.*]], float [[TMP10:%.*]], float [[TMP11:%.*]], float [[TMP12:%.*]], float [[TMP13:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[ADDR:%.*]] = call i64 @_AmdGetResumePointAddr() #[[ATTR3]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA2:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA]], i64 [[ADDR]], 5 -; LOWERRAYTRACINGPIPELINE-NEXT: [[NEWDATA:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @_AmdWaitAwaitTraversal(i64 4, i64 -1, [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA2]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_CallShader( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i32 [[TMP0:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[NEWDATA:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @_AmdAwaitShader(i64 2, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define i1 @_cont_ReportHit.struct.BuiltInTriangleIntersectionAttributes( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], float [[T:%.*]], i32 [[HITKIND:%.*]], ptr [[TMP0:%.*]]) #[[ATTR2:[0-9]+]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: [[DOANYHIT:%.*]] = fcmp fast ogt float [[T]], 0.000000e+00 -; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[DOANYHIT]], label [[ANYHIT:%.*]], label [[ACCEPTHIT:%.*]] -; LOWERRAYTRACINGPIPELINE: anyhit: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = load [[STRUCT_TRAVERSALDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP0]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = call ptr inttoptr (i64 3 to ptr)([[STRUCT_TRAVERSALDATA]] [[TRAV_DATA]], float [[T]], i32 [[HITKIND]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP3]]), !continuation.registercount !33, !continuation.returnedRegistercount !33 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = call [[STRUCT_TRAVERSALDATA]] @await.struct.TraversalData(ptr [[TMP4]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_TRAVERSALDATA]] [[TMP5]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[DATA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP6]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemDataAnyHit(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret i1 true -; LOWERRAYTRACINGPIPELINE: accepthit: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP9]], ptr [[TMP8]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP10]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP12]], ptr [[TMP11]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP2]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[DATA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP14]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP13]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret i1 true -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define i1 @_cont_ReportHit.struct.LargeIntersectionAttributes( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], float [[T:%.*]], i32 [[HITKIND:%.*]], ptr [[TMP0:%.*]]) #[[ATTR2]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: [[DOANYHIT:%.*]] = fcmp fast ogt float [[T]], 0.000000e+00 -; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[DOANYHIT]], label [[ANYHIT:%.*]], label [[ACCEPTHIT:%.*]] -; LOWERRAYTRACINGPIPELINE: anyhit: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = load [[STRUCT_TRAVERSALDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = load [[STRUCT_LARGEINTERSECTIONATTRIBUTES:%.*]], ptr [[TMP0]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = call ptr inttoptr (i64 3 to ptr)([[STRUCT_TRAVERSALDATA]] [[TRAV_DATA]], float [[T]], i32 [[HITKIND]], [[STRUCT_LARGEINTERSECTIONATTRIBUTES]] [[TMP3]]), !continuation.registercount !33, !continuation.returnedRegistercount !33 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = call [[STRUCT_TRAVERSALDATA]] @await.struct.TraversalData(ptr [[TMP4]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_TRAVERSALDATA]] [[TMP5]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[DATA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP6]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemDataAnyHit(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret i1 true -; LOWERRAYTRACINGPIPELINE: accepthit: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP9]], ptr [[TMP8]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP10]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP12]], ptr [[TMP11]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = load i32, ptr [[TMP13]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP14]], ptr getelementptr inbounds ([30 x i32], ptr @PAYLOAD, i32 0, i32 1), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 3 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP16]], ptr getelementptr inbounds ([30 x i32], ptr @PAYLOAD, i32 0, i64 2), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP18]], ptr getelementptr inbounds ([30 x i32], ptr @PAYLOAD, i32 0, i64 3), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 5 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP19]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP20]], ptr getelementptr inbounds ([30 x i32], ptr @PAYLOAD, i32 0, i64 4), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 6 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP22]], ptr getelementptr inbounds ([30 x i32], ptr @PAYLOAD, i32 0, i64 5), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP23:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP2]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP24:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[DATA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP24]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP23]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret i1 true -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define i1 @_cont_ReportHit( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], float [[T:%.*]], i32 [[HITKIND:%.*]]) #[[ATTR2]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DOANYHIT:%.*]] = fcmp fast ogt float [[T]], 0.000000e+00 -; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[DOANYHIT]], label [[ANYHIT:%.*]], label [[ACCEPTHIT:%.*]] -; LOWERRAYTRACINGPIPELINE: anyhit: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = load [[STRUCT_TRAVERSALDATA:%.*]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[NEWDATA:%.*]] = call [[STRUCT_TRAVERSALDATA]] @_AmdAwaitAnyHit(i64 3, [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA]], float [[T]], i32 [[HITKIND]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_TRAVERSALDATA]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[DATA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP1]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemDataAnyHit(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret i1 true -; LOWERRAYTRACINGPIPELINE: accepthit: -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdAcceptHitAttributes(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret i1 true -; -; ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.HitData @_cont_GetCandidateState( ; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]]) #[[ATTR0]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA:%.*]], ptr [[DATA]], i32 0, i32 1 @@ -576,41 +408,81 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; ; ; LOWERRAYTRACINGPIPELINE-LABEL: define void @MyRayGen( -; LOWERRAYTRACINGPIPELINE-SAME: ) #[[ATTR4:[0-9]+]] !lgc.rt.shaderstage !23 !continuation.entry !14 !continuation.registercount !23 !continuation !36 { +; LOWERRAYTRACINGPIPELINE-SAME: ) #[[ATTR3:[0-9]+]] !lgc.rt.shaderstage [[META23:![0-9]+]] !continuation.entry [[META14:![0-9]+]] !continuation.registercount [[META23]] !continuation [[META35:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP1:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @continuations.getSystemData.s_struct.DispatchSystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP1]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = load [[DX_TYPES_HANDLE:%.*]], ptr @"\01?Scene@@3URaytracingAccelerationStructure@@A", align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = load [[DX_TYPES_HANDLE]], ptr @"\01?RenderTarget@@3V?$RWTexture2D@V?$vector@M$03@@@@A", align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = alloca [[STRUCT_RAYPAYLOAD:%.*]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = bitcast ptr [[TMP4]] to ptr ; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr [[TMP5]]) #[[ATTR1:[0-9]+]] ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP4]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: store <4 x float> zeroinitializer, ptr [[TMP6]], align 4, !tbaa [[TBAA37:![0-9]+]] +; LOWERRAYTRACINGPIPELINE-NEXT: store <4 x float> zeroinitializer, ptr [[TMP6]], align 4, !tbaa [[TBAA36:![0-9]+]] ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP2]]) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP7]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[TMP8]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_TraceRay.struct.RayPayload.attr_max_32_bytes(ptr [[SYSTEM_DATA_ALLOCA]], i64 [[TMP9]], i32 16, i32 -1, i32 0, i32 1, i32 0, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0x3F50624DE0000000, float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 1.000000e+04, ptr [[TMP4]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = load <4 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA37]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() -; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT:%.*]] = extractelement <3 x i32> [[TMP11]], i8 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() -; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT1:%.*]] = extractelement <3 x i32> [[TMP12]], i8 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP3]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP13]], [[DX_TYPES_RESOURCEPROPERTIES]] { i32 4098, i32 1033 }) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = extractelement <4 x float> [[TMP10]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = extractelement <4 x float> [[TMP10]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = extractelement <4 x float> [[TMP10]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = extractelement <4 x float> [[TMP10]], i64 3 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @dx.op.textureStore.f32(i32 67, [[DX_TYPES_HANDLE]] [[TMP14]], i32 [[EXTRACT]], i32 [[EXTRACT1]], i32 undef, float [[TMP15]], float [[TMP16]], float [[TMP17]], float [[TMP18]], i8 15) +; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA_I:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA_I:%.*]] = insertvalue [[STRUCT_SYSTEMDATA:%.*]] undef, [[STRUCT_DISPATCHSYSTEMDATA]] [[DIS_DATA_I]], 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA_I:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA:%.*]] undef, [[STRUCT_SYSTEMDATA]] [[SYS_DATA_I]], 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[ADDR_I:%.*]] = call i64 @_AmdGetResumePointAddr() #[[ATTR2:[0-9]+]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA2_I:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]], i64 [[ADDR_I]], 5 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP4]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[TMP10]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[TMP11]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP13]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[TMP10]], i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[TMP14]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP16]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = getelementptr i32, ptr [[TMP14]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP18]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[TMP14]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP19]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP20]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP21:%.*]] = call ptr inttoptr (i64 4 to ptr)(i64 -1, [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA2_I]]), !continuation.registercount [[META39:![0-9]+]], !continuation.returnedRegistercount !39, !continuation.wait.await [[META14]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP22:%.*]] = call [[STRUCT_DISPATCHSYSTEMDATA]] @await.struct.DispatchSystemData(ptr [[TMP21]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_RAYPAYLOAD]] poison, ptr [[TMP4]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP23:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP4]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP24:%.*]] = getelementptr i32, ptr [[TMP23]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr [[TMP24]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP26:%.*]] = load i32, ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP26]], ptr [[TMP25]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP27:%.*]] = getelementptr i32, ptr [[TMP23]], i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr [[TMP27]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP29]], ptr [[TMP28]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP30:%.*]] = getelementptr i32, ptr [[TMP27]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP31:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP31]], ptr [[TMP30]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr [[TMP27]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP33:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP33]], ptr [[TMP32]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP22]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; LOWERRAYTRACINGPIPELINE-NEXT: br label [[DOTSPLIT:%.*]] +; LOWERRAYTRACINGPIPELINE: .split: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = load <4 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA36]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP35:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() +; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT:%.*]] = extractelement <3 x i32> [[TMP35]], i8 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP36:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() +; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT1:%.*]] = extractelement <3 x i32> [[TMP36]], i8 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP37:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP3]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP37]], [[DX_TYPES_RESOURCEPROPERTIES]] { i32 4098, i32 1033 }) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = extractelement <4 x float> [[TMP34]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP40:%.*]] = extractelement <4 x float> [[TMP34]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP41:%.*]] = extractelement <4 x float> [[TMP34]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP42:%.*]] = extractelement <4 x float> [[TMP34]], i64 3 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @dx.op.textureStore.f32(i32 67, [[DX_TYPES_HANDLE]] [[TMP38]], i32 [[EXTRACT]], i32 [[EXTRACT1]], i32 undef, float [[TMP39]], float [[TMP40]], float [[TMP41]], float [[TMP42]], i8 15) ; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.end.p0(i64 16, ptr [[TMP5]]) #[[ATTR1]] -; LOWERRAYTRACINGPIPELINE-NEXT: ret void, !continuation.registercount !33 +; LOWERRAYTRACINGPIPELINE-NEXT: ret void, !continuation.registercount [[META33:![0-9]+]] ; ; ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.DispatchSystemData @MyClosestHitShader( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR4]] !lgc.rt.shaderstage !40 !continuation.registercount !35 !continuation !41 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR3]] !lgc.rt.shaderstage [[META40:![0-9]+]] !continuation.registercount [[META39]] !continuation [[META41:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_RAYPAYLOAD:%.*]], align 8 @@ -618,8 +490,7 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = call [[STRUCT_SYSTEMDATA]] @continuations.getSystemData.s_struct.SystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_SYSTEMDATA]] [[TMP4]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP3]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[TMP7]], i64 0 @@ -636,51 +507,52 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP16]], ptr [[TMP15]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = call [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] @_cont_GetTriangleHitAttributes(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP17]], ptr [[TMP2]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[HITATTRS]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP19]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP20]], ptr [[TMP18]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[HITATTRS]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP23]], ptr [[TMP21]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP24:%.*]] = getelementptr inbounds [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[HITATTRS]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP25:%.*]] = load <2 x float>, ptr [[TMP24]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP26:%.*]] = extractelement <2 x float> [[TMP25]], i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP27:%.*]] = fsub fast float 1.000000e+00, [[TMP26]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = extractelement <2 x float> [[TMP25]], i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = fsub fast float [[TMP27]], [[TMP28]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP30:%.*]] = insertelement <4 x float> undef, float [[TMP29]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP31:%.*]] = insertelement <4 x float> [[TMP30]], float [[TMP26]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP32:%.*]] = insertelement <4 x float> [[TMP31]], float [[TMP28]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP33:%.*]] = insertelement <4 x float> [[TMP32]], float 1.000000e+00, i64 3 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP3]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: store <4 x float> [[TMP33]], ptr [[TMP34]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[ADDR_I:%.*]] = getelementptr [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[VAL_I:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[ADDR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[VAL_I]], ptr [[TMP2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[HITATTRS]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP18]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP19]], ptr [[TMP17]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[HITATTRS]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP22]], ptr [[TMP20]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP23:%.*]] = getelementptr inbounds [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[HITATTRS]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP24:%.*]] = load <2 x float>, ptr [[TMP23]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP25:%.*]] = extractelement <2 x float> [[TMP24]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP26:%.*]] = fsub fast float 1.000000e+00, [[TMP25]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP27:%.*]] = extractelement <2 x float> [[TMP24]], i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = fsub fast float [[TMP26]], [[TMP27]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = insertelement <4 x float> undef, float [[TMP28]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP30:%.*]] = insertelement <4 x float> [[TMP29]], float [[TMP25]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP31:%.*]] = insertelement <4 x float> [[TMP30]], float [[TMP27]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP32:%.*]] = insertelement <4 x float> [[TMP31]], float 1.000000e+00, i64 3 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP33:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP3]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: store <4 x float> [[TMP32]], ptr [[TMP33]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP35:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP3]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP36:%.*]] = getelementptr i32, ptr [[TMP35]], i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP37:%.*]] = getelementptr i32, ptr [[TMP36]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = load i32, ptr [[TMP37]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP38]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = getelementptr i32, ptr [[TMP35]], i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP40:%.*]] = getelementptr i32, ptr [[TMP39]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP41:%.*]] = load i32, ptr [[TMP40]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP41]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP42:%.*]] = getelementptr i32, ptr [[TMP39]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP43:%.*]] = load i32, ptr [[TMP42]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP43]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP44:%.*]] = getelementptr i32, ptr [[TMP39]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP45:%.*]] = load i32, ptr [[TMP44]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP45]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP46:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP47:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP46]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP47]], !continuation.registercount !35 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP3]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP35:%.*]] = getelementptr i32, ptr [[TMP34]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP36:%.*]] = getelementptr i32, ptr [[TMP35]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP37:%.*]] = load i32, ptr [[TMP36]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP37]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = getelementptr i32, ptr [[TMP34]], i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = getelementptr i32, ptr [[TMP38]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP40:%.*]] = load i32, ptr [[TMP39]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP40]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP41:%.*]] = getelementptr i32, ptr [[TMP38]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP42:%.*]] = load i32, ptr [[TMP41]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP42]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP43:%.*]] = getelementptr i32, ptr [[TMP38]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP44:%.*]] = load i32, ptr [[TMP43]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP44]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP45:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP46:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP45]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP46]], !continuation.registercount [[META39]] ; ; ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.TraversalData @MyAnyHitShader( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_TRAVERSALDATA:%.*]] [[TMP0:%.*]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] [[TMP1:%.*]]) #[[ATTR4]] !lgc.rt.shaderstage !42 !continuation.registercount !35 !continuation !43 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_TRAVERSALDATA:%.*]] [[TMP0:%.*]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] [[TMP1:%.*]]) #[[ATTR3]] !lgc.rt.shaderstage [[META42:![0-9]+]] !continuation.registercount [[META39]] !continuation [[META43:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_HITDATA:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = alloca [[STRUCT_HITDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = alloca [[STRUCT_HITDATA]], align 8 @@ -694,8 +566,7 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = call [[STRUCT_TRAVERSALDATA]] @continuations.getSystemData.s_struct.TraversalDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_TRAVERSALDATA]] [[TMP10]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP11]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP9]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[TMP12]], i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[TMP13]], i64 0 @@ -713,183 +584,255 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP22]], ptr [[TMP21]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP23:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP24:%.*]] = call [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] @_cont_GetTriangleHitAttributes(ptr [[TMP23]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP24]], ptr [[TMP8]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP26]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP27]], ptr [[TMP25]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP30:%.*]] = load i32, ptr [[TMP29]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP30]], ptr [[TMP28]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[ADDR_I:%.*]] = getelementptr [[STRUCT_SYSTEMDATA:%.*]], ptr [[TMP23]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[VAL_I:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[ADDR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[VAL_I]], ptr [[TMP8]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP25]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP26]], ptr [[TMP24]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[ORIGHITATTRS]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = load i32, ptr [[TMP28]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP29]], ptr [[TMP27]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP1]], ptr [[HITATTRSALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP31:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP9]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP32:%.*]] = load <4 x float>, ptr [[TMP31]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP33:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP34]], ptr [[TMP4]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP35:%.*]] = call <3 x float> @_cont_ObjectRayOrigin3(ptr [[TMP33]], ptr [[TMP4]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT1:%.*]] = extractelement <3 x float> [[TMP35]], i8 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP30:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP9]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP31:%.*]] = load <4 x float>, ptr [[TMP30]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP32:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_I3:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I4:%.*]] = load [[STRUCT_HITDATA]], ptr [[RESPTR_I3]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[RES_I4]], ptr [[TMP4]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP33:%.*]] = call <3 x float> @_cont_ObjectRayOrigin3(ptr [[TMP32]], ptr [[TMP4]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT1:%.*]] = extractelement <3 x float> [[TMP33]], i8 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_I:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I:%.*]] = load [[STRUCT_HITDATA]], ptr [[RESPTR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[RES_I]], ptr [[TMP5]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP35:%.*]] = call <3 x float> @_cont_ObjectRayDirection3(ptr [[TMP34]], ptr [[TMP5]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT:%.*]] = extractelement <3 x float> [[TMP35]], i8 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP36:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP37:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP37]], ptr [[TMP5]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = call <3 x float> @_cont_ObjectRayDirection3(ptr [[TMP36]], ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT:%.*]] = extractelement <3 x float> [[TMP38]], i8 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP40:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP40]], ptr [[TMP3]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP41:%.*]] = call float @_cont_RayTCurrent(ptr [[TMP39]], ptr [[TMP3]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP42:%.*]] = fmul fast float [[TMP41]], [[EXTRACT]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP43:%.*]] = fadd fast float [[TMP42]], [[EXTRACT1]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP44:%.*]] = fcmp fast ogt float [[TMP43]], 0.000000e+00 -; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[TMP44]], label [[TMP45:%.*]], label [[TMP67:%.*]] -; LOWERRAYTRACINGPIPELINE: 45: -; LOWERRAYTRACINGPIPELINE-NEXT: store <4 x float> [[TMP32]], ptr [[TMP31]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP46:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_AcceptHitAndEndSearch(ptr [[TMP46]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_I5:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I6:%.*]] = load [[STRUCT_HITDATA]], ptr [[RESPTR_I5]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[RES_I6]], ptr [[TMP3]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I7:%.*]] = load float, ptr [[TMP3]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP37:%.*]] = fmul fast float [[RES_I7]], [[EXTRACT]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = fadd fast float [[TMP37]], [[EXTRACT1]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = fcmp fast ogt float [[TMP38]], 0.000000e+00 +; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[TMP39]], label [[TMP40:%.*]], label [[TMP62:%.*]] +; LOWERRAYTRACINGPIPELINE: 40: +; LOWERRAYTRACINGPIPELINE-NEXT: store <4 x float> [[TMP31]], ptr [[TMP30]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP41:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_AcceptHitAndEndSearch(ptr [[TMP41]]) ; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP47:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP9]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP48:%.*]] = getelementptr i32, ptr [[TMP47]], i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP49:%.*]] = getelementptr i32, ptr [[TMP48]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP42:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP9]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP43:%.*]] = getelementptr i32, ptr [[TMP42]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP44:%.*]] = getelementptr i32, ptr [[TMP43]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP45:%.*]] = load i32, ptr [[TMP44]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP45]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP46:%.*]] = getelementptr i32, ptr [[TMP42]], i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP47:%.*]] = getelementptr i32, ptr [[TMP46]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP48:%.*]] = load i32, ptr [[TMP47]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP48]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP49:%.*]] = getelementptr i32, ptr [[TMP46]], i64 1 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP50:%.*]] = load i32, ptr [[TMP49]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP50]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP51:%.*]] = getelementptr i32, ptr [[TMP47]], i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP52:%.*]] = getelementptr i32, ptr [[TMP51]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP53:%.*]] = load i32, ptr [[TMP52]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP53]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP54:%.*]] = getelementptr i32, ptr [[TMP51]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP55:%.*]] = load i32, ptr [[TMP54]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP55]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP56:%.*]] = getelementptr i32, ptr [[TMP51]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP57:%.*]] = load i32, ptr [[TMP56]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP57]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP58:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP59:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP60:%.*]] = load i32, ptr [[TMP58]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP60]], ptr [[TMP59]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP61:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP62:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP63:%.*]] = load i32, ptr [[TMP61]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP63]], ptr [[TMP62]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP64:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP7]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP65:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP65]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP64]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP66:%.*]] = load [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_TRAVERSALDATA]] [[TMP66]], !continuation.registercount !35 -; LOWERRAYTRACINGPIPELINE: 67: -; LOWERRAYTRACINGPIPELINE-NEXT: store <4 x float> [[TMP32]], ptr [[TMP31]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP50]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP51:%.*]] = getelementptr i32, ptr [[TMP46]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP52:%.*]] = load i32, ptr [[TMP51]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP52]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_3_CLOSESTHIT_IN_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP53:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP54:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP55:%.*]] = load i32, ptr [[TMP53]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP55]], ptr [[TMP54]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP56:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP57:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP58:%.*]] = load i32, ptr [[TMP56]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP58]], ptr [[TMP57]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP59:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP7]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP60:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[ADDR_I1:%.*]] = getelementptr [[STRUCT_SYSTEMDATA]], ptr [[TMP60]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP59]], ptr [[ADDR_I1]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP61:%.*]] = load [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_TRAVERSALDATA]] [[TMP61]], !continuation.registercount [[META39]] +; LOWERRAYTRACINGPIPELINE: 62: +; LOWERRAYTRACINGPIPELINE-NEXT: store <4 x float> [[TMP31]], ptr [[TMP30]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_AcceptHit(ptr [[SYSTEM_DATA_ALLOCA]]) ; LOWERRAYTRACINGPIPELINE-NEXT: call void (...) @registerbuffer.setpointerbarrier(ptr @PAYLOAD) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP68:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP9]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP69:%.*]] = getelementptr i32, ptr [[TMP68]], i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP70:%.*]] = getelementptr i32, ptr [[TMP69]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP63:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP9]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP64:%.*]] = getelementptr i32, ptr [[TMP63]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP65:%.*]] = getelementptr i32, ptr [[TMP64]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP66:%.*]] = load i32, ptr [[TMP65]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP66]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP67:%.*]] = getelementptr i32, ptr [[TMP63]], i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP68:%.*]] = getelementptr i32, ptr [[TMP67]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP69:%.*]] = load i32, ptr [[TMP68]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP69]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP70:%.*]] = getelementptr i32, ptr [[TMP67]], i64 1 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP71:%.*]] = load i32, ptr [[TMP70]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP71]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP72:%.*]] = getelementptr i32, ptr [[TMP68]], i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP73:%.*]] = getelementptr i32, ptr [[TMP72]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP74:%.*]] = load i32, ptr [[TMP73]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP74]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP75:%.*]] = getelementptr i32, ptr [[TMP72]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP76:%.*]] = load i32, ptr [[TMP75]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP76]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP77:%.*]] = getelementptr i32, ptr [[TMP72]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP78:%.*]] = load i32, ptr [[TMP77]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP78]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP79:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP80:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP81:%.*]] = load i32, ptr [[TMP79]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP81]], ptr [[TMP80]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP82:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP83:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP84:%.*]] = load i32, ptr [[TMP82]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP84]], ptr [[TMP83]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP85:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP6]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP86:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_SetTriangleHitAttributes(ptr [[TMP86]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP85]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP87:%.*]] = load [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_TRAVERSALDATA]] [[TMP87]], !continuation.registercount !35 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP71]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP72:%.*]] = getelementptr i32, ptr [[TMP67]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP73:%.*]] = load i32, ptr [[TMP72]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP73]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_2_ANYHIT_OUT_ACCEPT_PAYLOAD_ATTR_0_I32S]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP74:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP75:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP76:%.*]] = load i32, ptr [[TMP74]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP76]], ptr [[TMP75]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP77:%.*]] = getelementptr inbounds i32, ptr [[HITATTRSALLOCA]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP78:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP79:%.*]] = load i32, ptr [[TMP77]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP79]], ptr [[TMP78]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP80:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP6]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP81:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[ADDR_I2:%.*]] = getelementptr [[STRUCT_SYSTEMDATA]], ptr [[TMP81]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP80]], ptr [[ADDR_I2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP82:%.*]] = load [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_TRAVERSALDATA]] [[TMP82]], !continuation.registercount [[META39]] ; ; ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.TraversalData @MyIntersectionShader( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_TRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR4]] !lgc.rt.shaderstage !44 !continuation.registercount !33 !continuation !45 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_TRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR3]] !lgc.rt.shaderstage [[META44:![0-9]+]] !continuation.registercount [[META33]] !continuation [[META45:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_HITDATA:%.*]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_TRAVERSALDATA]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = call [[STRUCT_TRAVERSALDATA]] @continuations.getSystemData.s_struct.TraversalDatas() -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_TRAVERSALDATA]] [[TMP4]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = call [[STRUCT_TRAVERSALDATA]] @continuations.getSystemData.s_struct.TraversalDatas() +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_TRAVERSALDATA]] [[TMP5]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP7]], ptr [[TMP2]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = call float @_cont_RayTCurrent(ptr [[TMP6]], ptr [[TMP2]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = bitcast ptr [[TMP3]] to ptr -; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[TMP9]]) #[[ATTR1]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = call i1 @_cont_ReportHit.struct.BuiltInTriangleIntersectionAttributes(ptr [[SYSTEM_DATA_ALLOCA]], float [[TMP8]], i32 0, ptr [[TMP3]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = call i1 @_cont_IsEndSearch(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP14:%.*]] -; LOWERRAYTRACINGPIPELINE: 12: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = load [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_TRAVERSALDATA]] [[TMP13]], !continuation.registercount !33 -; LOWERRAYTRACINGPIPELINE: 14: -; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[TMP9]]) #[[ATTR1]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = load [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_TRAVERSALDATA]] [[TMP15]], !continuation.registercount !33 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_I:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I:%.*]] = load [[STRUCT_HITDATA]], ptr [[RESPTR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[RES_I]], ptr [[TMP2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I1:%.*]] = load float, ptr [[TMP2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = bitcast ptr [[TMP4]] to ptr +; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[TMP8]]) #[[ATTR1]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[DOANYHIT_I:%.*]] = fcmp fast ogt float [[RES_I1]], 0.000000e+00 +; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[DOANYHIT_I]], label [[ANYHIT_I:%.*]], label [[ACCEPTHIT_I:%.*]] +; LOWERRAYTRACINGPIPELINE: anyhit.i: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA_I:%.*]] = load [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP4]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = call ptr inttoptr (i64 3 to ptr)([[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]], float [[RES_I1]], i32 0, [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP9]]), !continuation.registercount [[META33]], !continuation.returnedRegistercount !33 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = call [[STRUCT_TRAVERSALDATA]] @await.struct.TraversalData(ptr [[TMP10]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_TRAVERSALDATA]] [[TMP11]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; LOWERRAYTRACINGPIPELINE-NEXT: br label [[_CONT_REPORTHIT_EXIT:%.*]] +; LOWERRAYTRACINGPIPELINE: accepthit.i: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP13]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP15]], ptr [[TMP14]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP16]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP18]], ptr [[TMP17]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP19:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP3]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP20:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[ADDR_I:%.*]] = getelementptr [[STRUCT_SYSTEMDATA:%.*]], ptr [[TMP20]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP19]], ptr [[ADDR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: br label [[_CONT_REPORTHIT_EXIT]] +; LOWERRAYTRACINGPIPELINE: _cont_ReportHit.exit: +; LOWERRAYTRACINGPIPELINE-NEXT: [[ISEND_I:%.*]] = call i1 @opaqueIsEnd() +; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[ISEND_I]], label [[TMP21:%.*]], label [[TMP23:%.*]] +; LOWERRAYTRACINGPIPELINE: 21: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP22:%.*]] = load [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_TRAVERSALDATA]] [[TMP22]], !continuation.registercount [[META33]] +; LOWERRAYTRACINGPIPELINE: 23: +; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[TMP8]]) #[[ATTR1]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP24:%.*]] = load [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_TRAVERSALDATA]] [[TMP24]], !continuation.registercount [[META33]] ; ; ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.TraversalData @MyIntersectionShaderLargeAttrs( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_TRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR4]] !lgc.rt.shaderstage !44 !continuation.registercount !33 !continuation !46 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_TRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR3]] !lgc.rt.shaderstage [[META44]] !continuation.registercount [[META33]] !continuation [[META46:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_HITDATA:%.*]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_LARGEINTERSECTIONATTRIBUTES:%.*]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = alloca [[STRUCT_LARGEINTERSECTIONATTRIBUTES:%.*]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_TRAVERSALDATA]], align 8 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = call [[STRUCT_TRAVERSALDATA]] @continuations.getSystemData.s_struct.TraversalDatas() -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_TRAVERSALDATA]] [[TMP4]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP5]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = call [[STRUCT_TRAVERSALDATA]] @continuations.getSystemData.s_struct.TraversalDatas() +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_TRAVERSALDATA]] [[TMP5]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = call [[STRUCT_HITDATA]] @_cont_GetCandidateState(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[TMP7]], ptr [[TMP2]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = call float @_cont_RayTCurrent(ptr [[TMP6]], ptr [[TMP2]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[PTR0:%.*]] = getelementptr [[STRUCT_LARGEINTERSECTIONATTRIBUTES]], ptr [[TMP3]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_I:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I:%.*]] = load [[STRUCT_HITDATA]], ptr [[RESPTR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_HITDATA]] [[RES_I]], ptr [[TMP2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[RES_I1:%.*]] = load float, ptr [[TMP2]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[PTR0:%.*]] = getelementptr [[STRUCT_LARGEINTERSECTIONATTRIBUTES]], ptr [[TMP4]], i32 0, i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 100, ptr [[PTR0]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[PTR1:%.*]] = getelementptr [[STRUCT_LARGEINTERSECTIONATTRIBUTES]], ptr [[TMP3]], i32 0, i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[PTR1:%.*]] = getelementptr [[STRUCT_LARGEINTERSECTIONATTRIBUTES]], ptr [[TMP4]], i32 0, i32 0, i32 1 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 101, ptr [[PTR1]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[PTR2:%.*]] = getelementptr [[STRUCT_LARGEINTERSECTIONATTRIBUTES]], ptr [[TMP3]], i32 0, i32 0, i32 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[PTR2:%.*]] = getelementptr [[STRUCT_LARGEINTERSECTIONATTRIBUTES]], ptr [[TMP4]], i32 0, i32 0, i32 2 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 102, ptr [[PTR2]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[PTR3:%.*]] = getelementptr [[STRUCT_LARGEINTERSECTIONATTRIBUTES]], ptr [[TMP3]], i32 0, i32 0, i32 3 +; LOWERRAYTRACINGPIPELINE-NEXT: [[PTR3:%.*]] = getelementptr [[STRUCT_LARGEINTERSECTIONATTRIBUTES]], ptr [[TMP4]], i32 0, i32 0, i32 3 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 103, ptr [[PTR3]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[PTR4:%.*]] = getelementptr [[STRUCT_LARGEINTERSECTIONATTRIBUTES]], ptr [[TMP3]], i32 0, i32 0, i32 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[PTR4:%.*]] = getelementptr [[STRUCT_LARGEINTERSECTIONATTRIBUTES]], ptr [[TMP4]], i32 0, i32 0, i32 4 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 104, ptr [[PTR4]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[PTR5:%.*]] = getelementptr [[STRUCT_LARGEINTERSECTIONATTRIBUTES]], ptr [[TMP3]], i32 0, i32 0, i32 5 +; LOWERRAYTRACINGPIPELINE-NEXT: [[PTR5:%.*]] = getelementptr [[STRUCT_LARGEINTERSECTIONATTRIBUTES]], ptr [[TMP4]], i32 0, i32 0, i32 5 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 105, ptr [[PTR5]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[PTR6:%.*]] = getelementptr [[STRUCT_LARGEINTERSECTIONATTRIBUTES]], ptr [[TMP3]], i32 0, i32 0, i32 6 +; LOWERRAYTRACINGPIPELINE-NEXT: [[PTR6:%.*]] = getelementptr [[STRUCT_LARGEINTERSECTIONATTRIBUTES]], ptr [[TMP4]], i32 0, i32 0, i32 6 ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 106, ptr [[PTR6]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = bitcast ptr [[TMP3]] to ptr -; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[TMP9]]) #[[ATTR1]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = call i1 @_cont_ReportHit.struct.LargeIntersectionAttributes(ptr [[SYSTEM_DATA_ALLOCA]], float [[TMP8]], i32 0, ptr [[TMP3]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = call i1 @_cont_IsEndSearch(ptr [[SYSTEM_DATA_ALLOCA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP14:%.*]] -; LOWERRAYTRACINGPIPELINE: 12: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = load [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_TRAVERSALDATA]] [[TMP13]], !continuation.registercount !33 -; LOWERRAYTRACINGPIPELINE: 14: -; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[TMP9]]) #[[ATTR1]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = load [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_TRAVERSALDATA]] [[TMP15]], !continuation.registercount !33 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = bitcast ptr [[TMP4]] to ptr +; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[TMP8]]) #[[ATTR1]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[DOANYHIT_I:%.*]] = fcmp fast ogt float [[RES_I1]], 0.000000e+00 +; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[DOANYHIT_I]], label [[ANYHIT_I:%.*]], label [[ACCEPTHIT_I:%.*]] +; LOWERRAYTRACINGPIPELINE: anyhit.i: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA_I:%.*]] = load [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = load [[STRUCT_LARGEINTERSECTIONATTRIBUTES]], ptr [[TMP4]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = call ptr inttoptr (i64 3 to ptr)([[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I]], float [[RES_I1]], i32 0, [[STRUCT_LARGEINTERSECTIONATTRIBUTES]] [[TMP9]]), !continuation.registercount [[META33]], !continuation.returnedRegistercount !33 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = call [[STRUCT_TRAVERSALDATA]] @await.struct.TraversalData(ptr [[TMP10]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_TRAVERSALDATA]] [[TMP11]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) +; LOWERRAYTRACINGPIPELINE-NEXT: br label [[_CONT_REPORTHIT_EXIT:%.*]] +; LOWERRAYTRACINGPIPELINE: accepthit.i: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP13]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP15]], ptr [[TMP14]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP16]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP18]], ptr [[TMP17]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP19]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP20]], ptr getelementptr inbounds ([30 x i32], ptr @PAYLOAD, i32 0, i32 1), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 3 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP22]], ptr getelementptr inbounds ([30 x i32], ptr @PAYLOAD, i32 0, i64 2), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP24]], ptr getelementptr inbounds ([30 x i32], ptr @PAYLOAD, i32 0, i64 3), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 5 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP25]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP26]], ptr getelementptr inbounds ([30 x i32], ptr @PAYLOAD, i32 0, i64 4), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 6 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = load i32, ptr [[TMP27]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP28]], ptr getelementptr inbounds ([30 x i32], ptr @PAYLOAD, i32 0, i64 5), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = load [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[TMP3]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP30:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[ADDR_I:%.*]] = getelementptr [[STRUCT_SYSTEMDATA:%.*]], ptr [[TMP30]], i32 0, i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP29]], ptr [[ADDR_I]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: br label [[_CONT_REPORTHIT_EXIT]] +; LOWERRAYTRACINGPIPELINE: _cont_ReportHit.exit: +; LOWERRAYTRACINGPIPELINE-NEXT: [[ISEND_I:%.*]] = call i1 @opaqueIsEnd() +; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[ISEND_I]], label [[TMP31:%.*]], label [[TMP33:%.*]] +; LOWERRAYTRACINGPIPELINE: 31: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP32:%.*]] = load [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_TRAVERSALDATA]] [[TMP32]], !continuation.registercount [[META33]] +; LOWERRAYTRACINGPIPELINE: 33: +; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[TMP8]]) #[[ATTR1]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = load [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_TRAVERSALDATA]] [[TMP34]], !continuation.registercount [[META33]] ; ; ; LOWERRAYTRACINGPIPELINE-LABEL: define %struct.DispatchSystemData @MyMissShader( -; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR4]] !lgc.rt.shaderstage !47 !continuation.registercount !35 !continuation !48 { +; LOWERRAYTRACINGPIPELINE-SAME: [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR3]] !lgc.rt.shaderstage [[META47:![0-9]+]] !continuation.registercount [[META39]] !continuation [[META48:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_RAYPAYLOAD:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = call [[STRUCT_SYSTEMDATA]] @continuations.getSystemData.s_struct.SystemDatas() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_SYSTEMDATA]] [[TMP3]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP4]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP2]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[TMP5]], i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i64 0 @@ -926,7 +869,7 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP27]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = getelementptr inbounds [[STRUCT_SYSTEMDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = load [[STRUCT_DISPATCHSYSTEMDATA:%.*]], ptr [[TMP28]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP29]], !continuation.registercount !35 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[STRUCT_DISPATCHSYSTEMDATA]] [[TMP29]], !continuation.registercount [[META39]] ; ; ; DXILCONTPOSTPROCESS-LABEL: define i1 @_cont_IsEndSearch( @@ -969,7 +912,7 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; ; ; DXILCONTPOSTPROCESS-LABEL: define void @MyRayGen( -; DXILCONTPOSTPROCESS-SAME: ) #[[ATTR3:[0-9]+]] !lgc.rt.shaderstage !22 !continuation.entry !13 !continuation.registercount !22 !continuation !34 !continuation.state !22 { +; DXILCONTPOSTPROCESS-SAME: ) #[[ATTR3:[0-9]+]] !lgc.rt.shaderstage [[META22:![0-9]+]] !continuation.entry [[META13:![0-9]+]] !continuation.registercount [[META22]] !continuation [[META34:![0-9]+]] !continuation.state [[META22]] { ; DXILCONTPOSTPROCESS-NEXT: AllocaSpillBB: ; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA:%.*]], align 8 ; DXILCONTPOSTPROCESS-NEXT: [[CONT_STATE:%.*]] = alloca [0 x i32], align 4 @@ -1004,12 +947,12 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; DXILCONTPOSTPROCESS-NEXT: [[TMP12:%.*]] = bitcast float [[DOTSROA_0_12_VEC_EXTRACT]] to i32 ; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP12]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 9) to ptr addrspace(20)), align 4 ; DXILCONTPOSTPROCESS-NEXT: [[TMP13:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: call void (i64, i64, ...) @continuation.waitContinue(i64 4, i64 -1, i32 [[TMP13]], [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA2_I]]), !continuation.registercount !35, !continuation.returnedRegistercount !35 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, i64, ...) @continuation.waitContinue(i64 4, i64 -1, i32 [[TMP13]], [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA2_I]]), !continuation.registercount [[META35:![0-9]+]], !continuation.returnedRegistercount !35 ; DXILCONTPOSTPROCESS-NEXT: unreachable ; ; -; DXILCONTPOSTPROCESS-LABEL: define void @MyRayGen.resume.0( -; DXILCONTPOSTPROCESS-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage !22 !continuation.registercount !35 !continuation !34 { +; DXILCONTPOSTPROCESS-LABEL: define dso_local void @MyRayGen.resume.0( +; DXILCONTPOSTPROCESS-SAME: i32 [[TMP0:%.*]], [[STRUCT_DISPATCHSYSTEMDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage [[META22]] !continuation.registercount [[META35]] !continuation [[META34]] { ; DXILCONTPOSTPROCESS-NEXT: entryresume.0: ; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_DISPATCHSYSTEMDATA]], align 8 ; DXILCONTPOSTPROCESS-NEXT: [[CONT_STATE:%.*]] = alloca [0 x i32], align 4 @@ -1045,7 +988,7 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; ; ; DXILCONTPOSTPROCESS-LABEL: define void @MyClosestHitShader( -; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR3]] !lgc.rt.shaderstage !36 !continuation.registercount !35 !continuation !37 !continuation.state !22 { +; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR3]] !lgc.rt.shaderstage [[META36:![0-9]+]] !continuation.registercount [[META35]] !continuation [[META37:![0-9]+]] !continuation.state [[META22]] { ; DXILCONTPOSTPROCESS-NEXT: AllocaSpillBB: ; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 ; DXILCONTPOSTPROCESS-NEXT: [[CONT_STATE:%.*]] = alloca [0 x i32], align 4 @@ -1100,12 +1043,12 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP25]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 9) to ptr addrspace(20)), align 4 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA:%.*]] poison, <3 x i32> [[DOTFCA_0_0_EXTRACT]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[TMP26:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP26]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount !35 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP26]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount [[META35]] ; DXILCONTPOSTPROCESS-NEXT: unreachable ; ; ; DXILCONTPOSTPROCESS-LABEL: define void @MyAnyHitShader( -; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_TRAVERSALDATA:%.*]] [[TMP0:%.*]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] [[TMP1:%.*]]) #[[ATTR3]] !lgc.rt.shaderstage !38 !continuation.registercount !35 !continuation !39 !continuation.state !22 { +; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_TRAVERSALDATA:%.*]] [[TMP0:%.*]], [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] [[TMP1:%.*]]) #[[ATTR3]] !lgc.rt.shaderstage [[META38:![0-9]+]] !continuation.registercount [[META35]] !continuation [[META39:![0-9]+]] !continuation.state [[META22]] { ; DXILCONTPOSTPROCESS-NEXT: AllocaSpillBB: ; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_TRAVERSALDATA]], align 8 ; DXILCONTPOSTPROCESS-NEXT: [[TMP2:%.*]] = alloca [[STRUCT_HITDATA:%.*]], align 8 @@ -1166,6 +1109,22 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; DXILCONTPOSTPROCESS-NEXT: [[TMP16:%.*]] = bitcast float [[DOTSROA_060_4_VEC_EXTRACT]] to i32 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[TMP1]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[RESPTR_I3:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I4_FCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[RESPTR_I3]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I4_FCA_0_LOAD:%.*]] = load float, ptr [[RES_I4_FCA_0_GEP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I4_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] poison, float [[RES_I4_FCA_0_LOAD]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I4_FCA_1_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[RESPTR_I3]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I4_FCA_1_LOAD:%.*]] = load i32, ptr [[RES_I4_FCA_1_GEP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I4_FCA_1_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] [[RES_I4_FCA_0_INSERT]], i32 [[RES_I4_FCA_1_LOAD]], 1 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I4_FCA_1_INSERT_FCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I4_FCA_1_INSERT]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I4_FCA_1_INSERT_FCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[TMP2]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: store float [[RES_I4_FCA_1_INSERT_FCA_0_EXTRACT]], ptr [[RES_I4_FCA_1_INSERT_FCA_0_GEP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I4_FCA_1_INSERT_FCA_1_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I4_FCA_1_INSERT]], 1 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I4_FCA_1_INSERT_FCA_1_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[TMP2]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[RES_I4_FCA_1_INSERT_FCA_1_EXTRACT]], ptr [[RES_I4_FCA_1_INSERT_FCA_1_GEP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP18:%.*]] = call <3 x float> @_cont_ObjectRayOrigin3(ptr [[TMP17]], ptr [[TMP2]]) +; DXILCONTPOSTPROCESS-NEXT: [[EXTRACT1:%.*]] = extractelement <3 x float> [[TMP18]], i8 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP19:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: [[RESPTR_I:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 1 ; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[RESPTR_I]], i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_0_LOAD:%.*]] = load float, ptr [[RES_I_FCA_0_GEP]], align 4 @@ -1174,40 +1133,24 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_1_LOAD:%.*]] = load i32, ptr [[RES_I_FCA_1_GEP]], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_1_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] [[RES_I_FCA_0_INSERT]], i32 [[RES_I_FCA_1_LOAD]], 1 ; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_1_INSERT_FCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I_FCA_1_INSERT]], 0 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_1_INSERT_FCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[TMP2]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_1_INSERT_FCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[TMP3]], i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: store float [[RES_I_FCA_1_INSERT_FCA_0_EXTRACT]], ptr [[RES_I_FCA_1_INSERT_FCA_0_GEP]], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_1_INSERT_FCA_1_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I_FCA_1_INSERT]], 1 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_1_INSERT_FCA_1_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[TMP2]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_1_INSERT_FCA_1_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[TMP3]], i32 0, i32 1 ; DXILCONTPOSTPROCESS-NEXT: store i32 [[RES_I_FCA_1_INSERT_FCA_1_EXTRACT]], ptr [[RES_I_FCA_1_INSERT_FCA_1_GEP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP18:%.*]] = call <3 x float> @_cont_ObjectRayOrigin3(ptr [[TMP17]], ptr [[TMP2]]) -; DXILCONTPOSTPROCESS-NEXT: [[EXTRACT1:%.*]] = extractelement <3 x float> [[TMP18]], i8 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP19:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[RESPTR_I1:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I2_FCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[RESPTR_I1]], i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I2_FCA_0_LOAD:%.*]] = load float, ptr [[RES_I2_FCA_0_GEP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I2_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] poison, float [[RES_I2_FCA_0_LOAD]], 0 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I2_FCA_1_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[RESPTR_I1]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I2_FCA_1_LOAD:%.*]] = load i32, ptr [[RES_I2_FCA_1_GEP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I2_FCA_1_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] [[RES_I2_FCA_0_INSERT]], i32 [[RES_I2_FCA_1_LOAD]], 1 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I2_FCA_1_INSERT_FCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I2_FCA_1_INSERT]], 0 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I2_FCA_1_INSERT_FCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[TMP3]], i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: store float [[RES_I2_FCA_1_INSERT_FCA_0_EXTRACT]], ptr [[RES_I2_FCA_1_INSERT_FCA_0_GEP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I2_FCA_1_INSERT_FCA_1_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I2_FCA_1_INSERT]], 1 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I2_FCA_1_INSERT_FCA_1_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[TMP3]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[RES_I2_FCA_1_INSERT_FCA_1_EXTRACT]], ptr [[RES_I2_FCA_1_INSERT_FCA_1_GEP]], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[TMP20:%.*]] = call <3 x float> @_cont_ObjectRayDirection3(ptr [[TMP19]], ptr [[TMP3]]) ; DXILCONTPOSTPROCESS-NEXT: [[EXTRACT:%.*]] = extractelement <3 x float> [[TMP20]], i8 0 ; DXILCONTPOSTPROCESS-NEXT: [[TMP21:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[RESPTR_I3:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I4_FCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[RESPTR_I3]], i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I4_FCA_0_LOAD:%.*]] = load float, ptr [[RES_I4_FCA_0_GEP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I4_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] poison, float [[RES_I4_FCA_0_LOAD]], 0 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I4_FCA_1_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[RESPTR_I3]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I4_FCA_1_LOAD:%.*]] = load i32, ptr [[RES_I4_FCA_1_GEP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I4_FCA_1_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] [[RES_I4_FCA_0_INSERT]], i32 [[RES_I4_FCA_1_LOAD]], 1 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I4_FCA_1_INSERT_FCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I4_FCA_1_INSERT]], 0 -; DXILCONTPOSTPROCESS-NEXT: [[RES_I4_FCA_1_INSERT_FCA_1_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I4_FCA_1_INSERT]], 1 -; DXILCONTPOSTPROCESS-NEXT: [[TMP22:%.*]] = fmul fast float [[RES_I4_FCA_1_INSERT_FCA_0_EXTRACT]], [[EXTRACT]] +; DXILCONTPOSTPROCESS-NEXT: [[RESPTR_I5:%.*]] = getelementptr [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I6_FCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[RESPTR_I5]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I6_FCA_0_LOAD:%.*]] = load float, ptr [[RES_I6_FCA_0_GEP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I6_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] poison, float [[RES_I6_FCA_0_LOAD]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I6_FCA_1_GEP:%.*]] = getelementptr inbounds [[STRUCT_HITDATA]], ptr [[RESPTR_I5]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I6_FCA_1_LOAD:%.*]] = load i32, ptr [[RES_I6_FCA_1_GEP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I6_FCA_1_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] [[RES_I6_FCA_0_INSERT]], i32 [[RES_I6_FCA_1_LOAD]], 1 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I6_FCA_1_INSERT_FCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I6_FCA_1_INSERT]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[RES_I6_FCA_1_INSERT_FCA_1_EXTRACT:%.*]] = extractvalue [[STRUCT_HITDATA]] [[RES_I6_FCA_1_INSERT]], 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP22:%.*]] = fmul fast float [[RES_I6_FCA_1_INSERT_FCA_0_EXTRACT]], [[EXTRACT]] ; DXILCONTPOSTPROCESS-NEXT: [[TMP23:%.*]] = fadd fast float [[TMP22]], [[EXTRACT1]] ; DXILCONTPOSTPROCESS-NEXT: [[TMP24:%.*]] = fcmp fast ogt float [[TMP23]], 0.000000e+00 ; DXILCONTPOSTPROCESS-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP37:%.*]] @@ -1236,9 +1179,9 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; DXILCONTPOSTPROCESS-NEXT: [[DOTSROA_062_4_VEC_INSERT:%.*]] = insertelement <2 x float> [[DOTSROA_062_0_VEC_INSERT]], float [[TMP34]], i32 1 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] poison, <2 x float> [[DOTSROA_062_4_VEC_INSERT]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[TMP35:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[ADDR_I6:%.*]] = getelementptr [[STRUCT_SYSTEMDATA]], ptr [[TMP35]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[ADDR_I1:%.*]] = getelementptr [[STRUCT_SYSTEMDATA]], ptr [[TMP35]], i32 0, i32 1 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_EXTRACT25:%.*]] = extractvalue [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[DOTFCA_0_INSERT]], 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[ADDR_I6]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_GEP:%.*]] = getelementptr inbounds [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[ADDR_I1]], i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: store <2 x float> [[DOTFCA_0_EXTRACT25]], ptr [[DOTFCA_0_GEP]], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_GEP26:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_LOAD:%.*]] = load <3 x i32>, ptr [[DOTFCA_0_0_0_GEP26]], align 4 @@ -1265,7 +1208,7 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_LOAD:%.*]] = load i64, ptr [[DOTFCA_5_GEP33]], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_4_INSERT]], i64 [[DOTFCA_5_LOAD]], 5 ; DXILCONTPOSTPROCESS-NEXT: [[TMP36:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP36]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT]]), !continuation.registercount !35 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP36]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT]]), !continuation.registercount [[META35]] ; DXILCONTPOSTPROCESS-NEXT: unreachable ; DXILCONTPOSTPROCESS: 37: ; DXILCONTPOSTPROCESS-NEXT: call void @_cont_AcceptHit(ptr [[SYSTEM_DATA_ALLOCA]]) @@ -1291,9 +1234,9 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; DXILCONTPOSTPROCESS-NEXT: [[DOTSROA_066_4_VEC_INSERT:%.*]] = insertelement <2 x float> [[DOTSROA_066_0_VEC_INSERT]], float [[TMP45]], i32 1 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT65:%.*]] = insertvalue [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] poison, <2 x float> [[DOTSROA_066_4_VEC_INSERT]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[TMP46:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[ADDR_I7:%.*]] = getelementptr [[STRUCT_SYSTEMDATA]], ptr [[TMP46]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[ADDR_I2:%.*]] = getelementptr [[STRUCT_SYSTEMDATA]], ptr [[TMP46]], i32 0, i32 1 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_EXTRACT34:%.*]] = extractvalue [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[DOTFCA_0_INSERT65]], 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_GEP35:%.*]] = getelementptr inbounds [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[ADDR_I7]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_GEP35:%.*]] = getelementptr inbounds [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]], ptr [[ADDR_I2]], i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: store <2 x float> [[DOTFCA_0_EXTRACT34]], ptr [[DOTFCA_0_GEP35]], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_GEP36:%.*]] = getelementptr inbounds [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0, i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_LOAD37:%.*]] = load <3 x i32>, ptr [[DOTFCA_0_0_0_GEP36]], align 4 @@ -1320,33 +1263,29 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_LOAD58:%.*]] = load i64, ptr [[DOTFCA_5_GEP57]], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_INSERT59:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_4_INSERT56]], i64 [[DOTFCA_5_LOAD58]], 5 ; DXILCONTPOSTPROCESS-NEXT: [[TMP47:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP47]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT59]]), !continuation.registercount !35 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP47]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT59]]), !continuation.registercount [[META35]] ; DXILCONTPOSTPROCESS-NEXT: unreachable ; ; ; DXILCONTPOSTPROCESS-LABEL: define void @MyIntersectionShader( -; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_TRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR3]] !lgc.rt.shaderstage !40 !continuation.registercount !32 !continuation !41 !continuation.state !42 !continuation.stacksize !42 { +; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_TRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR3]] !lgc.rt.shaderstage [[META40:![0-9]+]] !continuation.registercount [[META32:![0-9]+]] !continuation [[META41:![0-9]+]] !continuation.state [[META42:![0-9]+]] !continuation.stacksize [[META42]] { ; DXILCONTPOSTPROCESS-NEXT: AllocaSpillBB: ; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_TRAVERSALDATA]], align 8 ; DXILCONTPOSTPROCESS-NEXT: [[CONT_STATE:%.*]] = alloca [2 x i32], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[CSP:%.*]] = alloca i32, align 4 ; DXILCONTPOSTPROCESS-NEXT: store [[STRUCT_TRAVERSALDATA]] [[TMP0]], ptr [[SYSTEM_DATA]], align 4 ; DXILCONTPOSTPROCESS-NEXT: store i32 [[CSPINIT]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP1:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 0 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP2]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP1]] to ptr addrspace(21) ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_SPILL_ADDR:%.*]] = getelementptr inbounds [[MYINTERSECTIONSHADER_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: store i64 [[RETURNADDR]], ptr [[RETURNADDR_SPILL_ADDR]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP4:%.*]] = load [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP4]], 0, 0, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP4]], 0, 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP4]], 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP4]], 1, 1 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP4]], 2 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP4]], 3 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP4]], 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP4]], 5 +; DXILCONTPOSTPROCESS-NEXT: [[TMP1:%.*]] = load [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 0, 0, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 0, 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 1, 1 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 2 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 3 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 5 ; DXILCONTPOSTPROCESS-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA:%.*]] poison, float [[DOTFCA_1_0_EXTRACT]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_1_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] [[RES_I_FCA_0_INSERT]], i32 [[DOTFCA_1_1_EXTRACT]], 1 @@ -1364,60 +1303,50 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; DXILCONTPOSTPROCESS-NEXT: [[TRAV_DATA_I_FCA_4_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I_FCA_3_INSERT]], float [[DOTFCA_4_EXTRACT]], 4 ; DXILCONTPOSTPROCESS-NEXT: [[TRAV_DATA_I_FCA_5_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I_FCA_4_INSERT]], i64 [[DOTFCA_5_EXTRACT]], 5 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] poison, <2 x float> undef, 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP5:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP6:%.*]] = add i32 [[TMP5]], 8 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP6]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP9:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP10:%.*]] = inttoptr i32 [[TMP9]] to ptr addrspace(21) -; DXILCONTPOSTPROCESS-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP10]], i64 -8 -; DXILCONTPOSTPROCESS-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP11]], i32 0 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP8]], ptr addrspace(21) [[TMP12]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: [[TMP14:%.*]] = load i32, ptr [[TMP13]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP15:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP16:%.*]] = inttoptr i32 [[TMP15]] to ptr addrspace(21) -; DXILCONTPOSTPROCESS-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP16]], i64 -8 -; DXILCONTPOSTPROCESS-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP17]], i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP19:%.*]] = getelementptr [2 x i32], ptr addrspace(21) [[TMP18]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP14]], ptr addrspace(21) [[TMP19]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP20:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP21:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @MyIntersectionShader.resume.0 to i64)) -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 3, i32 [[TMP20]], i64 [[TMP21]], [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I_FCA_5_INSERT]], float [[RES_I_FCA_1_INSERT_FCA_0_EXTRACT]], i32 0, [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[DOTFCA_0_INSERT]]), !continuation.registercount !32, !continuation.returnedRegistercount !32 +; DXILCONTPOSTPROCESS-NEXT: [[TMP2:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP3:%.*]] = add i32 [[TMP2]], 8 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP3]], ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP4:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP5]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP7]], ptr addrspace(21) [[TMP6]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP8]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP10]], ptr addrspace(21) [[TMP9]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP11:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP12:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @MyIntersectionShader.resume.0 to i64)) +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 3, i32 [[TMP11]], i64 [[TMP12]], [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I_FCA_5_INSERT]], float [[RES_I_FCA_1_INSERT_FCA_0_EXTRACT]], i32 0, [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[DOTFCA_0_INSERT]]), !continuation.registercount [[META32]], !continuation.returnedRegistercount !32 ; DXILCONTPOSTPROCESS-NEXT: unreachable ; DXILCONTPOSTPROCESS: accepthit.i: ; DXILCONTPOSTPROCESS-NEXT: [[DOTSROA_0_0_VEC_EXTRACT:%.*]] = extractelement <2 x float> undef, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP22:%.*]] = bitcast float [[DOTSROA_0_0_VEC_EXTRACT]] to i32 -; DXILCONTPOSTPROCESS-NEXT: [[TMP23:%.*]] = bitcast i32 [[TMP22]] to float -; DXILCONTPOSTPROCESS-NEXT: [[DOTSROA_066_0_VEC_INSERT:%.*]] = insertelement <2 x float> undef, float [[TMP23]], i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP13:%.*]] = bitcast float [[DOTSROA_0_0_VEC_EXTRACT]] to i32 +; DXILCONTPOSTPROCESS-NEXT: [[TMP14:%.*]] = bitcast i32 [[TMP13]] to float +; DXILCONTPOSTPROCESS-NEXT: [[DOTSROA_065_0_VEC_INSERT:%.*]] = insertelement <2 x float> undef, float [[TMP14]], i32 0 ; DXILCONTPOSTPROCESS-NEXT: [[DOTSROA_0_4_VEC_EXTRACT:%.*]] = extractelement <2 x float> undef, i32 1 -; DXILCONTPOSTPROCESS-NEXT: [[TMP24:%.*]] = bitcast float [[DOTSROA_0_4_VEC_EXTRACT]] to i32 -; DXILCONTPOSTPROCESS-NEXT: [[TMP25:%.*]] = bitcast i32 [[TMP24]] to float -; DXILCONTPOSTPROCESS-NEXT: [[DOTSROA_066_4_VEC_INSERT:%.*]] = insertelement <2 x float> [[DOTSROA_066_0_VEC_INSERT]], float [[TMP25]], i32 1 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT65:%.*]] = insertvalue [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] poison, <2 x float> [[DOTSROA_066_4_VEC_INSERT]], 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[DOTFCA_0_INSERT65]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP15:%.*]] = bitcast float [[DOTSROA_0_4_VEC_EXTRACT]] to i32 +; DXILCONTPOSTPROCESS-NEXT: [[TMP16:%.*]] = bitcast i32 [[TMP15]] to float +; DXILCONTPOSTPROCESS-NEXT: [[DOTSROA_065_4_VEC_INSERT:%.*]] = insertelement <2 x float> [[DOTSROA_065_0_VEC_INSERT]], float [[TMP16]], i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT64:%.*]] = insertvalue [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] poison, <2 x float> [[DOTSROA_065_4_VEC_INSERT]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[DOTFCA_0_INSERT64]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[ISEND_I:%.*]] = call i1 @opaqueIsEnd() -; DXILCONTPOSTPROCESS-NEXT: br i1 [[ISEND_I]], label [[TMP26:%.*]], label [[TMP31:%.*]] -; DXILCONTPOSTPROCESS: 26: +; DXILCONTPOSTPROCESS-NEXT: br i1 [[ISEND_I]], label [[TMP17:%.*]], label [[TMP19:%.*]] +; DXILCONTPOSTPROCESS: 17: ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_RELOAD_ADDR1:%.*]] = getelementptr inbounds [[MYINTERSECTIONSHADER_FRAME]], ptr [[CONT_STATE]], i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_RELOAD2:%.*]] = load i64, ptr [[RETURNADDR_RELOAD_ADDR1]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_INSERT12:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] poison, <3 x i32> [[DOTFCA_0_0_0_EXTRACT]], 0, 0, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_INSERT15:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_0_0_INSERT12]], <2 x float> [[DOTFCA_0_EXTRACT]], 0, 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_INSERT18:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_1_0_INSERT15]], float [[DOTFCA_1_0_EXTRACT]], 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_INSERT21:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_0_INSERT18]], i32 [[DOTFCA_1_1_EXTRACT]], 1, 1 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_INSERT24:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_1_INSERT21]], <3 x float> [[DOTFCA_2_EXTRACT]], 2 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_INSERT27:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_2_INSERT24]], <3 x float> [[DOTFCA_3_EXTRACT]], 3 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_INSERT30:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_3_INSERT27]], float [[DOTFCA_4_EXTRACT]], 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_INSERT33:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_4_INSERT30]], i64 [[DOTFCA_5_EXTRACT]], 5 -; DXILCONTPOSTPROCESS-NEXT: [[TMP27:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP28:%.*]] = add i32 [[TMP27]], 0 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP28]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP29:%.*]] = inttoptr i32 [[TMP28]] to ptr addrspace(21) -; DXILCONTPOSTPROCESS-NEXT: [[TMP30:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD2]], i32 [[TMP30]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT33]]), !continuation.registercount !32 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_INSERT28:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] poison, <3 x i32> [[DOTFCA_0_0_0_EXTRACT]], 0, 0, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_INSERT31:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_0_0_INSERT28]], <2 x float> [[DOTFCA_0_EXTRACT]], 0, 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_INSERT34:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_1_0_INSERT31]], float [[DOTFCA_1_0_EXTRACT]], 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_INSERT37:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_0_INSERT34]], i32 [[DOTFCA_1_1_EXTRACT]], 1, 1 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_INSERT40:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_1_INSERT37]], <3 x float> [[DOTFCA_2_EXTRACT]], 2 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_INSERT43:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_2_INSERT40]], <3 x float> [[DOTFCA_3_EXTRACT]], 3 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_INSERT46:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_3_INSERT43]], float [[DOTFCA_4_EXTRACT]], 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_INSERT49:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_4_INSERT46]], i64 [[DOTFCA_5_EXTRACT]], 5 +; DXILCONTPOSTPROCESS-NEXT: [[TMP18:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD2]], i32 [[TMP18]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT49]]), !continuation.registercount [[META32]] ; DXILCONTPOSTPROCESS-NEXT: unreachable -; DXILCONTPOSTPROCESS: 31: +; DXILCONTPOSTPROCESS: 19: ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[MYINTERSECTIONSHADER_FRAME]], ptr [[CONT_STATE]], i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_RELOAD:%.*]] = load i64, ptr [[RETURNADDR_RELOAD_ADDR]], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] poison, <3 x i32> [[DOTFCA_0_0_0_EXTRACT]], 0, 0, 0 @@ -1428,113 +1357,93 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_2_INSERT]], <3 x float> [[DOTFCA_3_EXTRACT]], 3 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_3_INSERT]], float [[DOTFCA_4_EXTRACT]], 4 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_4_INSERT]], i64 [[DOTFCA_5_EXTRACT]], 5 -; DXILCONTPOSTPROCESS-NEXT: [[TMP32:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP33:%.*]] = add i32 [[TMP32]], 0 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP33]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP34:%.*]] = inttoptr i32 [[TMP33]] to ptr addrspace(21) -; DXILCONTPOSTPROCESS-NEXT: [[TMP35:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP35]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT]]), !continuation.registercount !32 +; DXILCONTPOSTPROCESS-NEXT: [[TMP20:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP20]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT]]), !continuation.registercount [[META32]] ; DXILCONTPOSTPROCESS-NEXT: unreachable ; ; -; DXILCONTPOSTPROCESS-LABEL: define void @MyIntersectionShader.resume.0( -; DXILCONTPOSTPROCESS-SAME: i32 [[TMP0:%.*]], [[STRUCT_TRAVERSALDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage !40 !continuation.registercount !32 !continuation !41 { +; DXILCONTPOSTPROCESS-LABEL: define dso_local void @MyIntersectionShader.resume.0( +; DXILCONTPOSTPROCESS-SAME: i32 [[TMP0:%.*]], [[STRUCT_TRAVERSALDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage [[META40]] !continuation.registercount [[META32]] !continuation [[META41]] { ; DXILCONTPOSTPROCESS-NEXT: entryresume.0: ; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_TRAVERSALDATA]], align 8 ; DXILCONTPOSTPROCESS-NEXT: [[CONT_STATE:%.*]] = alloca [2 x i32], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[CSP:%.*]] = alloca i32, align 4 ; DXILCONTPOSTPROCESS-NEXT: store [[STRUCT_TRAVERSALDATA]] [[TMP1]], ptr [[SYSTEM_DATA]], align 4 ; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP0]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP2:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP3:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP4:%.*]] = inttoptr i32 [[TMP3]] to ptr addrspace(21) -; DXILCONTPOSTPROCESS-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP4]], i64 -8 -; DXILCONTPOSTPROCESS-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP5]], i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(21) [[TMP6]], align 4 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP7]], ptr [[TMP2]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: [[TMP9:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP10:%.*]] = inttoptr i32 [[TMP9]] to ptr addrspace(21) -; DXILCONTPOSTPROCESS-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP10]], i64 -8 -; DXILCONTPOSTPROCESS-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP11]], i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP13:%.*]] = getelementptr [2 x i32], ptr addrspace(21) [[TMP12]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: [[TMP14:%.*]] = load i32, ptr addrspace(21) [[TMP13]], align 4 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP14]], ptr [[TMP8]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP15:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], -8 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP16]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_EXTRACT34:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 0, 0, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_EXTRACT36:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 0, 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_EXTRACT38:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_EXTRACT40:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 1, 1 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_EXTRACT42:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 2 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_EXTRACT44:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 3 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_EXTRACT46:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_EXTRACT48:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 5 +; DXILCONTPOSTPROCESS-NEXT: [[TMP2:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP3]], i64 -8 +; DXILCONTPOSTPROCESS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(21) [[TMP5]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP7]], ptr [[TMP6]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP10:%.*]] = load i32, ptr addrspace(21) [[TMP8]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP10]], ptr [[TMP9]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP11:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], -8 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP12]], ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_EXTRACT10:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 0, 0, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_EXTRACT12:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 0, 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_EXTRACT14:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_EXTRACT16:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 1, 1 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_EXTRACT18:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 2 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_EXTRACT20:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 3 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_EXTRACT22:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_EXTRACT24:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 5 ; DXILCONTPOSTPROCESS-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; DXILCONTPOSTPROCESS-NEXT: [[ISEND_I:%.*]] = call i1 @opaqueIsEnd() -; DXILCONTPOSTPROCESS-NEXT: br i1 [[ISEND_I]], label [[TMP17:%.*]], label [[TMP22:%.*]] -; DXILCONTPOSTPROCESS: 17: +; DXILCONTPOSTPROCESS-NEXT: br i1 [[ISEND_I]], label [[TMP13:%.*]], label [[TMP15:%.*]] +; DXILCONTPOSTPROCESS: 13: ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_RELOAD_ADDR1:%.*]] = getelementptr inbounds [[MYINTERSECTIONSHADER_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_RELOAD2:%.*]] = load i64, ptr [[RETURNADDR_RELOAD_ADDR1]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_INSERT12:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] poison, <3 x i32> [[DOTFCA_0_0_0_EXTRACT34]], 0, 0, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_INSERT15:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_0_0_INSERT12]], <2 x float> [[DOTFCA_0_1_0_EXTRACT36]], 0, 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_INSERT18:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_1_0_INSERT15]], float [[DOTFCA_1_0_EXTRACT38]], 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_INSERT21:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_0_INSERT18]], i32 [[DOTFCA_1_1_EXTRACT40]], 1, 1 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_INSERT24:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_1_INSERT21]], <3 x float> [[DOTFCA_2_EXTRACT42]], 2 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_INSERT27:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_2_INSERT24]], <3 x float> [[DOTFCA_3_EXTRACT44]], 3 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_INSERT30:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_3_INSERT27]], float [[DOTFCA_4_EXTRACT46]], 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_INSERT33:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_4_INSERT30]], i64 [[DOTFCA_5_EXTRACT48]], 5 -; DXILCONTPOSTPROCESS-NEXT: [[TMP18:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP19:%.*]] = add i32 [[TMP18]], 0 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP19]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP20:%.*]] = inttoptr i32 [[TMP19]] to ptr addrspace(21) -; DXILCONTPOSTPROCESS-NEXT: [[TMP21:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD2]], i32 [[TMP21]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT33]]), !continuation.registercount !32 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_INSERT28:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] poison, <3 x i32> [[DOTFCA_0_0_0_EXTRACT10]], 0, 0, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_INSERT31:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_0_0_INSERT28]], <2 x float> [[DOTFCA_0_1_0_EXTRACT12]], 0, 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_INSERT34:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_1_0_INSERT31]], float [[DOTFCA_1_0_EXTRACT14]], 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_INSERT37:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_0_INSERT34]], i32 [[DOTFCA_1_1_EXTRACT16]], 1, 1 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_INSERT40:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_1_INSERT37]], <3 x float> [[DOTFCA_2_EXTRACT18]], 2 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_INSERT43:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_2_INSERT40]], <3 x float> [[DOTFCA_3_EXTRACT20]], 3 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_INSERT46:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_3_INSERT43]], float [[DOTFCA_4_EXTRACT22]], 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_INSERT49:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_4_INSERT46]], i64 [[DOTFCA_5_EXTRACT24]], 5 +; DXILCONTPOSTPROCESS-NEXT: [[TMP14:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD2]], i32 [[TMP14]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT49]]), !continuation.registercount [[META32]] ; DXILCONTPOSTPROCESS-NEXT: unreachable -; DXILCONTPOSTPROCESS: 22: +; DXILCONTPOSTPROCESS: 15: ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[MYINTERSECTIONSHADER_FRAME]], ptr [[CONT_STATE]], i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_RELOAD:%.*]] = load i64, ptr [[RETURNADDR_RELOAD_ADDR]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] poison, <3 x i32> [[DOTFCA_0_0_0_EXTRACT34]], 0, 0, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_0_0_INSERT]], <2 x float> [[DOTFCA_0_1_0_EXTRACT36]], 0, 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_1_0_INSERT]], float [[DOTFCA_1_0_EXTRACT38]], 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_0_INSERT]], i32 [[DOTFCA_1_1_EXTRACT40]], 1, 1 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_1_INSERT]], <3 x float> [[DOTFCA_2_EXTRACT42]], 2 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_2_INSERT]], <3 x float> [[DOTFCA_3_EXTRACT44]], 3 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_3_INSERT]], float [[DOTFCA_4_EXTRACT46]], 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_4_INSERT]], i64 [[DOTFCA_5_EXTRACT48]], 5 -; DXILCONTPOSTPROCESS-NEXT: [[TMP23:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP24:%.*]] = add i32 [[TMP23]], 0 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP24]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP25:%.*]] = inttoptr i32 [[TMP24]] to ptr addrspace(21) -; DXILCONTPOSTPROCESS-NEXT: [[TMP26:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP26]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT]]), !continuation.registercount !32 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] poison, <3 x i32> [[DOTFCA_0_0_0_EXTRACT10]], 0, 0, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_0_0_INSERT]], <2 x float> [[DOTFCA_0_1_0_EXTRACT12]], 0, 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_1_0_INSERT]], float [[DOTFCA_1_0_EXTRACT14]], 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_0_INSERT]], i32 [[DOTFCA_1_1_EXTRACT16]], 1, 1 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_1_INSERT]], <3 x float> [[DOTFCA_2_EXTRACT18]], 2 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_2_INSERT]], <3 x float> [[DOTFCA_3_EXTRACT20]], 3 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_3_INSERT]], float [[DOTFCA_4_EXTRACT22]], 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_4_INSERT]], i64 [[DOTFCA_5_EXTRACT24]], 5 +; DXILCONTPOSTPROCESS-NEXT: [[TMP16:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP16]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT]]), !continuation.registercount [[META32]] ; DXILCONTPOSTPROCESS-NEXT: unreachable ; ; ; DXILCONTPOSTPROCESS-LABEL: define void @MyIntersectionShaderLargeAttrs( -; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_TRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR3]] !lgc.rt.shaderstage !40 !continuation.registercount !32 !continuation !43 !continuation.state !42 !continuation.stacksize !42 { +; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_TRAVERSALDATA:%.*]] [[TMP0:%.*]]) #[[ATTR3]] !lgc.rt.shaderstage [[META40]] !continuation.registercount [[META32]] !continuation [[META43:![0-9]+]] !continuation.state [[META42]] !continuation.stacksize [[META42]] { ; DXILCONTPOSTPROCESS-NEXT: AllocaSpillBB: ; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_TRAVERSALDATA]], align 8 ; DXILCONTPOSTPROCESS-NEXT: [[CONT_STATE:%.*]] = alloca [2 x i32], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[CSP:%.*]] = alloca i32, align 4 ; DXILCONTPOSTPROCESS-NEXT: store [[STRUCT_TRAVERSALDATA]] [[TMP0]], ptr [[SYSTEM_DATA]], align 4 ; DXILCONTPOSTPROCESS-NEXT: store i32 [[CSPINIT]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP1:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 0 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP2]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP1]] to ptr addrspace(21) ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_SPILL_ADDR:%.*]] = getelementptr inbounds [[MYINTERSECTIONSHADERLARGEATTRS_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: store i64 [[RETURNADDR]], ptr [[RETURNADDR_SPILL_ADDR]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP4:%.*]] = load [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP4]], 0, 0, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP4]], 0, 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP4]], 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP4]], 1, 1 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP4]], 2 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP4]], 3 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP4]], 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP4]], 5 +; DXILCONTPOSTPROCESS-NEXT: [[TMP1:%.*]] = load [[STRUCT_TRAVERSALDATA]], ptr [[SYSTEM_DATA]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 0, 0, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 0, 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 1, 1 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 2 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 3 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_EXTRACT:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 5 ; DXILCONTPOSTPROCESS-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_0_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA:%.*]] poison, float [[DOTFCA_1_0_EXTRACT]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[RES_I_FCA_1_INSERT:%.*]] = insertvalue [[STRUCT_HITDATA]] [[RES_I_FCA_0_INSERT]], i32 [[DOTFCA_1_1_EXTRACT]], 1 @@ -1558,61 +1467,51 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_4_INSERT:%.*]] = insertvalue [[STRUCT_LARGEINTERSECTIONATTRIBUTES]] [[DOTFCA_0_3_INSERT]], i32 104, 0, 4 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_5_INSERT:%.*]] = insertvalue [[STRUCT_LARGEINTERSECTIONATTRIBUTES]] [[DOTFCA_0_4_INSERT]], i32 105, 0, 5 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_6_INSERT:%.*]] = insertvalue [[STRUCT_LARGEINTERSECTIONATTRIBUTES]] [[DOTFCA_0_5_INSERT]], i32 106, 0, 6 -; DXILCONTPOSTPROCESS-NEXT: [[TMP5:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP6:%.*]] = add i32 [[TMP5]], 8 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP6]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP7:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP9:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP10:%.*]] = inttoptr i32 [[TMP9]] to ptr addrspace(21) -; DXILCONTPOSTPROCESS-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP10]], i64 -8 -; DXILCONTPOSTPROCESS-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP11]], i32 0 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP8]], ptr addrspace(21) [[TMP12]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: [[TMP14:%.*]] = load i32, ptr [[TMP13]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP15:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP16:%.*]] = inttoptr i32 [[TMP15]] to ptr addrspace(21) -; DXILCONTPOSTPROCESS-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP16]], i64 -8 -; DXILCONTPOSTPROCESS-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP17]], i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP19:%.*]] = getelementptr [2 x i32], ptr addrspace(21) [[TMP18]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP14]], ptr addrspace(21) [[TMP19]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP20:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP21:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @MyIntersectionShaderLargeAttrs.resume.0 to i64)) -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 3, i32 [[TMP20]], i64 [[TMP21]], [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I_FCA_5_INSERT]], float [[RES_I_FCA_1_INSERT_FCA_0_EXTRACT]], i32 0, [[STRUCT_LARGEINTERSECTIONATTRIBUTES]] [[DOTFCA_0_6_INSERT]]), !continuation.registercount !32, !continuation.returnedRegistercount !32 +; DXILCONTPOSTPROCESS-NEXT: [[TMP2:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP3:%.*]] = add i32 [[TMP2]], 8 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP3]], ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP4:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP5]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP7]], ptr addrspace(21) [[TMP6]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP8]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP10]], ptr addrspace(21) [[TMP9]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP11:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP12:%.*]] = call i64 @continuation.getAddrAndMD(i64 ptrtoint (ptr @MyIntersectionShaderLargeAttrs.resume.0 to i64)) +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 3, i32 [[TMP11]], i64 [[TMP12]], [[STRUCT_TRAVERSALDATA]] [[TRAV_DATA_I_FCA_5_INSERT]], float [[RES_I_FCA_1_INSERT_FCA_0_EXTRACT]], i32 0, [[STRUCT_LARGEINTERSECTIONATTRIBUTES]] [[DOTFCA_0_6_INSERT]]), !continuation.registercount [[META32]], !continuation.returnedRegistercount !32 ; DXILCONTPOSTPROCESS-NEXT: unreachable ; DXILCONTPOSTPROCESS: accepthit.i: -; DXILCONTPOSTPROCESS-NEXT: [[TMP22:%.*]] = bitcast i32 100 to float -; DXILCONTPOSTPROCESS-NEXT: [[DOTSROA_071_0_VEC_INSERT:%.*]] = insertelement <2 x float> undef, float [[TMP22]], i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP23:%.*]] = bitcast i32 101 to float -; DXILCONTPOSTPROCESS-NEXT: [[DOTSROA_071_4_VEC_INSERT:%.*]] = insertelement <2 x float> [[DOTSROA_071_0_VEC_INSERT]], float [[TMP23]], i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP13:%.*]] = bitcast i32 100 to float +; DXILCONTPOSTPROCESS-NEXT: [[DOTSROA_070_0_VEC_INSERT:%.*]] = insertelement <2 x float> undef, float [[TMP13]], i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP14:%.*]] = bitcast i32 101 to float +; DXILCONTPOSTPROCESS-NEXT: [[DOTSROA_070_4_VEC_INSERT:%.*]] = insertelement <2 x float> [[DOTSROA_070_0_VEC_INSERT]], float [[TMP14]], i32 1 ; DXILCONTPOSTPROCESS-NEXT: store i32 102, ptr addrspace(20) addrspacecast (ptr getelementptr inbounds ([30 x i32], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 1) to ptr addrspace(20)), align 4 ; DXILCONTPOSTPROCESS-NEXT: store i32 103, ptr addrspace(20) addrspacecast (ptr getelementptr inbounds ([30 x i32], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i64 2) to ptr addrspace(20)), align 4 ; DXILCONTPOSTPROCESS-NEXT: store i32 104, ptr addrspace(20) addrspacecast (ptr getelementptr inbounds ([30 x i32], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i64 3) to ptr addrspace(20)), align 4 ; DXILCONTPOSTPROCESS-NEXT: store i32 105, ptr addrspace(20) addrspacecast (ptr getelementptr inbounds ([30 x i32], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i64 4) to ptr addrspace(20)), align 4 ; DXILCONTPOSTPROCESS-NEXT: store i32 106, ptr addrspace(20) addrspacecast (ptr getelementptr inbounds ([30 x i32], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i64 5) to ptr addrspace(20)), align 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] poison, <2 x float> [[DOTSROA_071_4_VEC_INSERT]], 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]] poison, <2 x float> [[DOTSROA_070_4_VEC_INSERT]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_EXTRACT:%.*]] = extractvalue [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES]] [[DOTFCA_0_INSERT]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[ISEND_I:%.*]] = call i1 @opaqueIsEnd() -; DXILCONTPOSTPROCESS-NEXT: br i1 [[ISEND_I]], label [[TMP24:%.*]], label [[TMP29:%.*]] -; DXILCONTPOSTPROCESS: 24: +; DXILCONTPOSTPROCESS-NEXT: br i1 [[ISEND_I]], label [[TMP15:%.*]], label [[TMP17:%.*]] +; DXILCONTPOSTPROCESS: 15: ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_RELOAD_ADDR1:%.*]] = getelementptr inbounds [[MYINTERSECTIONSHADERLARGEATTRS_FRAME]], ptr [[CONT_STATE]], i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_RELOAD2:%.*]] = load i64, ptr [[RETURNADDR_RELOAD_ADDR1]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_INSERT12:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] poison, <3 x i32> [[DOTFCA_0_0_0_EXTRACT]], 0, 0, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_INSERT15:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_0_0_INSERT12]], <2 x float> [[DOTFCA_0_EXTRACT]], 0, 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_INSERT18:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_1_0_INSERT15]], float [[DOTFCA_1_0_EXTRACT]], 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_INSERT21:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_0_INSERT18]], i32 [[DOTFCA_1_1_EXTRACT]], 1, 1 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_INSERT24:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_1_INSERT21]], <3 x float> [[DOTFCA_2_EXTRACT]], 2 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_INSERT27:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_2_INSERT24]], <3 x float> [[DOTFCA_3_EXTRACT]], 3 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_INSERT30:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_3_INSERT27]], float [[DOTFCA_4_EXTRACT]], 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_INSERT33:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_4_INSERT30]], i64 [[DOTFCA_5_EXTRACT]], 5 -; DXILCONTPOSTPROCESS-NEXT: [[TMP25:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP26:%.*]] = add i32 [[TMP25]], 0 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP26]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP27:%.*]] = inttoptr i32 [[TMP26]] to ptr addrspace(21) -; DXILCONTPOSTPROCESS-NEXT: [[TMP28:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD2]], i32 [[TMP28]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT33]]), !continuation.registercount !32 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_INSERT28:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] poison, <3 x i32> [[DOTFCA_0_0_0_EXTRACT]], 0, 0, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_INSERT31:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_0_0_INSERT28]], <2 x float> [[DOTFCA_0_EXTRACT]], 0, 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_INSERT34:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_1_0_INSERT31]], float [[DOTFCA_1_0_EXTRACT]], 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_INSERT37:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_0_INSERT34]], i32 [[DOTFCA_1_1_EXTRACT]], 1, 1 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_INSERT40:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_1_INSERT37]], <3 x float> [[DOTFCA_2_EXTRACT]], 2 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_INSERT43:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_2_INSERT40]], <3 x float> [[DOTFCA_3_EXTRACT]], 3 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_INSERT46:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_3_INSERT43]], float [[DOTFCA_4_EXTRACT]], 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_INSERT49:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_4_INSERT46]], i64 [[DOTFCA_5_EXTRACT]], 5 +; DXILCONTPOSTPROCESS-NEXT: [[TMP16:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD2]], i32 [[TMP16]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT49]]), !continuation.registercount [[META32]] ; DXILCONTPOSTPROCESS-NEXT: unreachable -; DXILCONTPOSTPROCESS: 29: +; DXILCONTPOSTPROCESS: 17: ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[MYINTERSECTIONSHADERLARGEATTRS_FRAME]], ptr [[CONT_STATE]], i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_RELOAD:%.*]] = load i64, ptr [[RETURNADDR_RELOAD_ADDR]], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] poison, <3 x i32> [[DOTFCA_0_0_0_EXTRACT]], 0, 0, 0 @@ -1623,92 +1522,76 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_2_INSERT]], <3 x float> [[DOTFCA_3_EXTRACT]], 3 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_3_INSERT]], float [[DOTFCA_4_EXTRACT]], 4 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_4_INSERT]], i64 [[DOTFCA_5_EXTRACT]], 5 -; DXILCONTPOSTPROCESS-NEXT: [[TMP30:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP31:%.*]] = add i32 [[TMP30]], 0 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP31]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP32:%.*]] = inttoptr i32 [[TMP31]] to ptr addrspace(21) -; DXILCONTPOSTPROCESS-NEXT: [[TMP33:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP33]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT]]), !continuation.registercount !32 +; DXILCONTPOSTPROCESS-NEXT: [[TMP18:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP18]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT]]), !continuation.registercount [[META32]] ; DXILCONTPOSTPROCESS-NEXT: unreachable ; ; -; DXILCONTPOSTPROCESS-LABEL: define void @MyIntersectionShaderLargeAttrs.resume.0( -; DXILCONTPOSTPROCESS-SAME: i32 [[TMP0:%.*]], [[STRUCT_TRAVERSALDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage !40 !continuation.registercount !32 !continuation !43 { +; DXILCONTPOSTPROCESS-LABEL: define dso_local void @MyIntersectionShaderLargeAttrs.resume.0( +; DXILCONTPOSTPROCESS-SAME: i32 [[TMP0:%.*]], [[STRUCT_TRAVERSALDATA:%.*]] [[TMP1:%.*]]) !lgc.rt.shaderstage [[META40]] !continuation.registercount [[META32]] !continuation [[META43]] { ; DXILCONTPOSTPROCESS-NEXT: entryresume.0: ; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_TRAVERSALDATA]], align 8 ; DXILCONTPOSTPROCESS-NEXT: [[CONT_STATE:%.*]] = alloca [2 x i32], align 4 ; DXILCONTPOSTPROCESS-NEXT: [[CSP:%.*]] = alloca i32, align 4 ; DXILCONTPOSTPROCESS-NEXT: store [[STRUCT_TRAVERSALDATA]] [[TMP1]], ptr [[SYSTEM_DATA]], align 4 ; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP0]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP2:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP3:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP4:%.*]] = inttoptr i32 [[TMP3]] to ptr addrspace(21) -; DXILCONTPOSTPROCESS-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP4]], i64 -8 -; DXILCONTPOSTPROCESS-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP5]], i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(21) [[TMP6]], align 4 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP7]], ptr [[TMP2]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: [[TMP9:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP10:%.*]] = inttoptr i32 [[TMP9]] to ptr addrspace(21) -; DXILCONTPOSTPROCESS-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP10]], i64 -8 -; DXILCONTPOSTPROCESS-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr addrspace(21) [[TMP11]], i32 0 -; DXILCONTPOSTPROCESS-NEXT: [[TMP13:%.*]] = getelementptr [2 x i32], ptr addrspace(21) [[TMP12]], i32 0, i32 1 -; DXILCONTPOSTPROCESS-NEXT: [[TMP14:%.*]] = load i32, ptr addrspace(21) [[TMP13]], align 4 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP14]], ptr [[TMP8]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP15:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], -8 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP16]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_EXTRACT34:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 0, 0, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_EXTRACT36:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 0, 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_EXTRACT38:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_EXTRACT40:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 1, 1 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_EXTRACT42:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 2 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_EXTRACT44:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 3 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_EXTRACT46:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_EXTRACT48:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 5 +; DXILCONTPOSTPROCESS-NEXT: [[TMP2:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr addrspace(21) +; DXILCONTPOSTPROCESS-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr addrspace(21) [[TMP3]], i64 -8 +; DXILCONTPOSTPROCESS-NEXT: [[TMP5:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 0 +; DXILCONTPOSTPROCESS-NEXT: [[TMP7:%.*]] = load i32, ptr addrspace(21) [[TMP5]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP7]], ptr [[TMP6]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i32], ptr addrspace(21) [[TMP4]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP9:%.*]] = getelementptr inbounds [2 x i32], ptr [[CONT_STATE]], i32 0, i32 1 +; DXILCONTPOSTPROCESS-NEXT: [[TMP10:%.*]] = load i32, ptr addrspace(21) [[TMP8]], align 4 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP10]], ptr [[TMP9]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP11:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], -8 +; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP12]], ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_EXTRACT10:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 0, 0, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_EXTRACT12:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 0, 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_EXTRACT14:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_EXTRACT16:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 1, 1 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_EXTRACT18:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 2 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_EXTRACT20:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 3 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_EXTRACT22:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_EXTRACT24:%.*]] = extractvalue [[STRUCT_TRAVERSALDATA]] [[TMP1]], 5 ; DXILCONTPOSTPROCESS-NEXT: call void @amd.dx.setLocalRootIndex(i32 5) ; DXILCONTPOSTPROCESS-NEXT: [[ISEND_I:%.*]] = call i1 @opaqueIsEnd() -; DXILCONTPOSTPROCESS-NEXT: br i1 [[ISEND_I]], label [[TMP17:%.*]], label [[TMP22:%.*]] -; DXILCONTPOSTPROCESS: 17: +; DXILCONTPOSTPROCESS-NEXT: br i1 [[ISEND_I]], label [[TMP13:%.*]], label [[TMP15:%.*]] +; DXILCONTPOSTPROCESS: 13: ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_RELOAD_ADDR1:%.*]] = getelementptr inbounds [[MYINTERSECTIONSHADERLARGEATTRS_FRAME:%.*]], ptr [[CONT_STATE]], i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_RELOAD2:%.*]] = load i64, ptr [[RETURNADDR_RELOAD_ADDR1]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_INSERT12:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] poison, <3 x i32> [[DOTFCA_0_0_0_EXTRACT34]], 0, 0, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_INSERT15:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_0_0_INSERT12]], <2 x float> [[DOTFCA_0_1_0_EXTRACT36]], 0, 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_INSERT18:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_1_0_INSERT15]], float [[DOTFCA_1_0_EXTRACT38]], 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_INSERT21:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_0_INSERT18]], i32 [[DOTFCA_1_1_EXTRACT40]], 1, 1 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_INSERT24:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_1_INSERT21]], <3 x float> [[DOTFCA_2_EXTRACT42]], 2 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_INSERT27:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_2_INSERT24]], <3 x float> [[DOTFCA_3_EXTRACT44]], 3 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_INSERT30:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_3_INSERT27]], float [[DOTFCA_4_EXTRACT46]], 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_INSERT33:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_4_INSERT30]], i64 [[DOTFCA_5_EXTRACT48]], 5 -; DXILCONTPOSTPROCESS-NEXT: [[TMP18:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP19:%.*]] = add i32 [[TMP18]], 0 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP19]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP20:%.*]] = inttoptr i32 [[TMP19]] to ptr addrspace(21) -; DXILCONTPOSTPROCESS-NEXT: [[TMP21:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD2]], i32 [[TMP21]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT33]]), !continuation.registercount !32 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_INSERT28:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] poison, <3 x i32> [[DOTFCA_0_0_0_EXTRACT10]], 0, 0, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_INSERT31:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_0_0_INSERT28]], <2 x float> [[DOTFCA_0_1_0_EXTRACT12]], 0, 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_INSERT34:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_1_0_INSERT31]], float [[DOTFCA_1_0_EXTRACT14]], 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_INSERT37:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_0_INSERT34]], i32 [[DOTFCA_1_1_EXTRACT16]], 1, 1 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_INSERT40:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_1_INSERT37]], <3 x float> [[DOTFCA_2_EXTRACT18]], 2 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_INSERT43:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_2_INSERT40]], <3 x float> [[DOTFCA_3_EXTRACT20]], 3 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_INSERT46:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_3_INSERT43]], float [[DOTFCA_4_EXTRACT22]], 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_INSERT49:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_4_INSERT46]], i64 [[DOTFCA_5_EXTRACT24]], 5 +; DXILCONTPOSTPROCESS-NEXT: [[TMP14:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD2]], i32 [[TMP14]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT49]]), !continuation.registercount [[META32]] ; DXILCONTPOSTPROCESS-NEXT: unreachable -; DXILCONTPOSTPROCESS: 22: +; DXILCONTPOSTPROCESS: 15: ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[MYINTERSECTIONSHADERLARGEATTRS_FRAME]], ptr [[CONT_STATE]], i32 0, i32 0 ; DXILCONTPOSTPROCESS-NEXT: [[RETURNADDR_RELOAD:%.*]] = load i64, ptr [[RETURNADDR_RELOAD_ADDR]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] poison, <3 x i32> [[DOTFCA_0_0_0_EXTRACT34]], 0, 0, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_0_0_INSERT]], <2 x float> [[DOTFCA_0_1_0_EXTRACT36]], 0, 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_1_0_INSERT]], float [[DOTFCA_1_0_EXTRACT38]], 1, 0 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_0_INSERT]], i32 [[DOTFCA_1_1_EXTRACT40]], 1, 1 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_1_INSERT]], <3 x float> [[DOTFCA_2_EXTRACT42]], 2 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_2_INSERT]], <3 x float> [[DOTFCA_3_EXTRACT44]], 3 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_3_INSERT]], float [[DOTFCA_4_EXTRACT46]], 4 -; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_4_INSERT]], i64 [[DOTFCA_5_EXTRACT48]], 5 -; DXILCONTPOSTPROCESS-NEXT: [[TMP23:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP24:%.*]] = add i32 [[TMP23]], 0 -; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP24]], ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: [[TMP25:%.*]] = inttoptr i32 [[TMP24]] to ptr addrspace(21) -; DXILCONTPOSTPROCESS-NEXT: [[TMP26:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP26]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT]]), !continuation.registercount !32 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_0_0_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] poison, <3 x i32> [[DOTFCA_0_0_0_EXTRACT10]], 0, 0, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_1_0_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_0_0_INSERT]], <2 x float> [[DOTFCA_0_1_0_EXTRACT12]], 0, 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_0_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_0_1_0_INSERT]], float [[DOTFCA_1_0_EXTRACT14]], 1, 0 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_1_1_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_0_INSERT]], i32 [[DOTFCA_1_1_EXTRACT16]], 1, 1 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_2_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_1_1_INSERT]], <3 x float> [[DOTFCA_2_EXTRACT18]], 2 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_3_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_2_INSERT]], <3 x float> [[DOTFCA_3_EXTRACT20]], 3 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_4_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_3_INSERT]], float [[DOTFCA_4_EXTRACT22]], 4 +; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_5_INSERT:%.*]] = insertvalue [[STRUCT_TRAVERSALDATA]] [[DOTFCA_4_INSERT]], i64 [[DOTFCA_5_EXTRACT24]], 5 +; DXILCONTPOSTPROCESS-NEXT: [[TMP16:%.*]] = load i32, ptr [[CSP]], align 4 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR_RELOAD]], i32 [[TMP16]], [[STRUCT_TRAVERSALDATA]] [[DOTFCA_5_INSERT]]), !continuation.registercount [[META32]] ; DXILCONTPOSTPROCESS-NEXT: unreachable ; ; ; DXILCONTPOSTPROCESS-LABEL: define void @MyMissShader( -; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR3]] !lgc.rt.shaderstage !44 !continuation.registercount !35 !continuation !45 !continuation.state !22 { +; DXILCONTPOSTPROCESS-SAME: i32 [[CSPINIT:%.*]], i64 [[RETURNADDR:%.*]], [[STRUCT_SYSTEMDATA:%.*]] [[TMP0:%.*]]) #[[ATTR3]] !lgc.rt.shaderstage [[META44:![0-9]+]] !continuation.registercount [[META35]] !continuation [[META45:![0-9]+]] !continuation.state [[META22]] { ; DXILCONTPOSTPROCESS-NEXT: AllocaSpillBB: ; DXILCONTPOSTPROCESS-NEXT: [[SYSTEM_DATA:%.*]] = alloca [[STRUCT_SYSTEMDATA]], align 8 ; DXILCONTPOSTPROCESS-NEXT: [[CONT_STATE:%.*]] = alloca [0 x i32], align 4 @@ -1745,6 +1628,6 @@ attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; DXILCONTPOSTPROCESS-NEXT: store i32 [[TMP13]], ptr addrspace(20) addrspacecast (ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_6_MISS_OUT]], ptr addrspacecast (ptr addrspace(20) @REGISTERS to ptr), i32 0, i32 0, i64 9) to ptr addrspace(20)), align 4 ; DXILCONTPOSTPROCESS-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue [[STRUCT_DISPATCHSYSTEMDATA:%.*]] poison, <3 x i32> [[DOTFCA_0_0_EXTRACT]], 0 ; DXILCONTPOSTPROCESS-NEXT: [[TMP14:%.*]] = load i32, ptr [[CSP]], align 4 -; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP14]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount !35 +; DXILCONTPOSTPROCESS-NEXT: call void (i64, ...) @continuation.continue(i64 [[RETURNADDR]], i32 [[TMP14]], [[STRUCT_DISPATCHSYSTEMDATA]] [[DOTFCA_0_INSERT]]), !continuation.registercount [[META35]] ; DXILCONTPOSTPROCESS-NEXT: unreachable ; diff --git a/shared/continuations/test/dx/unnamed-type-intrinsics.ll b/shared/continuations/test/dx/unnamed-type-intrinsics.ll index 16c88ea8e9..0aab765fec 100644 --- a/shared/continuations/test/dx/unnamed-type-intrinsics.ll +++ b/shared/continuations/test/dx/unnamed-type-intrinsics.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --include-generated-funcs --version 3 -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2>%t.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint,remove-types-metadata' -S %s 2> %t.stderr | FileCheck -check-prefix=LOWERRAYTRACINGPIPELINE %s ; RUN: count 0 < %t.stderr ; Check that using unnamed types works well with generating intrinsic names @@ -296,99 +296,6 @@ attributes #5 = { nocallback nofree nosync nounwind willreturn memory(argmem: re !44 = !{!"function", i1 poison, i32 poison, float poison, i32 poison, !42} !45 = !{!"function", !"void", i64 poison, !46} !46 = !{i32 0, i8 poison} -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_TraceRay.struct.RayPayload.attr_max_32_bytes( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i64 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]], i32 [[TMP3:%.*]], i32 [[TMP4:%.*]], i32 [[TMP5:%.*]], float [[TMP6:%.*]], float [[TMP7:%.*]], float [[TMP8:%.*]], float [[TMP9:%.*]], float [[TMP10:%.*]], float [[TMP11:%.*]], float [[TMP12:%.*]], float [[TMP13:%.*]], ptr [[TMP14:%.*]]) #[[ATTR0:[0-9]+]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[TMP0]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA:%.*]] = insertvalue [[TMP2]] undef, [[TMP0]] [[DIS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = insertvalue [[TMP1]] undef, [[TMP2]] [[SYS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[ADDR:%.*]] = call i64 @_AmdGetResumePointAddr() #[[ATTR3:[0-9]+]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA2:%.*]] = insertvalue [[TMP1]] [[TRAV_DATA]], i64 [[ADDR]], 5 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD:%.*]], ptr [[TMP14]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = getelementptr i32, ptr [[TMP16]], i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[TMP17]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP18]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP19]], ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[TMP16]], i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[TMP20]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP22]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP23:%.*]] = getelementptr i32, ptr [[TMP20]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP24]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr [[TMP20]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP25]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP26]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP27:%.*]] = call ptr inttoptr (i64 4 to ptr)([[TMP1]] [[TRAV_DATA2]]), !continuation.registercount !20, !continuation.returnedRegistercount !20 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = call [[TMP0]] @await.(ptr [[TMP27]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_RAYPAYLOAD]] poison, ptr [[TMP14]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP14]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP30:%.*]] = getelementptr i32, ptr [[TMP29]], i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP31:%.*]] = getelementptr i32, ptr [[TMP30]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP32:%.*]] = load i32, ptr @PAYLOAD, align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP32]], ptr [[TMP31]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP33:%.*]] = getelementptr i32, ptr [[TMP29]], i32 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = getelementptr i32, ptr [[TMP33]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP35:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP35]], ptr [[TMP34]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP36:%.*]] = getelementptr i32, ptr [[TMP33]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP37:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP37]], ptr [[TMP36]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = getelementptr i32, ptr [[TMP33]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP39]], ptr [[TMP38]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: store [[TMP0]] [[TMP28]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_TraceRay( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i64 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]], i32 [[TMP3:%.*]], i32 [[TMP4:%.*]], i32 [[TMP5:%.*]], float [[TMP6:%.*]], float [[TMP7:%.*]], float [[TMP8:%.*]], float [[TMP9:%.*]], float [[TMP10:%.*]], float [[TMP11:%.*]], float [[TMP12:%.*]], float [[TMP13:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[TMP0]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA:%.*]] = insertvalue [[TMP2]] undef, [[TMP0]] [[DIS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = insertvalue [[TMP1]] undef, [[TMP2]] [[SYS_DATA]], 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[ADDR:%.*]] = call i64 @_AmdGetResumePointAddr() #[[ATTR3]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA2:%.*]] = insertvalue [[TMP1]] [[TRAV_DATA]], i64 [[ADDR]], 5 -; LOWERRAYTRACINGPIPELINE-NEXT: [[NEWDATA:%.*]] = call [[TMP0]] @_AmdAwaitTraversal(i64 4, [[TMP1]] [[TRAV_DATA2]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[TMP0]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define void @_cont_CallShader( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], i32 [[TMP0:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA:%.*]] = load [[TMP0]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[NEWDATA:%.*]] = call [[TMP0]] @_AmdAwaitShader(i64 2, [[TMP0]] [[DIS_DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[TMP0]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemData(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret void -; -; -; LOWERRAYTRACINGPIPELINE-LABEL: define i1 @_cont_ReportHit( -; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]], float [[T:%.*]], i32 [[HITKIND:%.*]]) #[[ATTR0]] { -; LOWERRAYTRACINGPIPELINE-NEXT: [[ORIGTPTR:%.*]] = getelementptr inbounds [[TMP3:%.*]], ptr [[DATA]], i32 0, i32 0, i32 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[ORIGT:%.*]] = load float, ptr [[ORIGTPTR]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[ISNOHIT:%.*]] = fcmp fast uge float [[T]], [[ORIGT]] -; LOWERRAYTRACINGPIPELINE-NEXT: br i1 [[ISNOHIT]], label [[ISEND:%.*]], label [[CALLAHIT:%.*]] -; LOWERRAYTRACINGPIPELINE: callAHit: -; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA:%.*]] = load [[TMP3]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[NEWDATA:%.*]] = call [[TMP3]] @_AmdAwaitAnyHit(i64 3, [[TMP3]] [[TRAV_DATA]], float [[T]], i32 [[HITKIND]]) -; LOWERRAYTRACINGPIPELINE-NEXT: store [[TMP3]] [[NEWDATA]], ptr [[DATA]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[TMP3]], ptr [[DATA]], i32 0, i32 0, i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[TMP1]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdRestoreSystemDataAnyHit(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret i1 true -; LOWERRAYTRACINGPIPELINE: isEnd: -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_AmdAcceptHitAttributes(ptr [[DATA]]) -; LOWERRAYTRACINGPIPELINE-NEXT: ret i1 false -; -; ; LOWERRAYTRACINGPIPELINE-LABEL: define <3 x i32> @_cont_DispatchRaysIndex3( ; LOWERRAYTRACINGPIPELINE-SAME: ptr [[DATA:%.*]]) { ; LOWERRAYTRACINGPIPELINE-NEXT: [[RESPTR_1:%.*]] = getelementptr [[TMP0:%.*]], ptr [[DATA]], i32 0, i32 0, i32 0 @@ -439,7 +346,7 @@ attributes #5 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; ; ; LOWERRAYTRACINGPIPELINE-LABEL: define void @MyRayGen( -; LOWERRAYTRACINGPIPELINE-SAME: ) #[[ATTR2:[0-9]+]] !lgc.rt.shaderstage !15 !continuation.entry !21 !continuation.registercount !15 !continuation !22 { +; LOWERRAYTRACINGPIPELINE-SAME: ) #[[ATTR2:[0-9]+]] !lgc.rt.shaderstage [[META15:![0-9]+]] !continuation.entry [[META20:![0-9]+]] !continuation.registercount [[META15]] !continuation [[META21:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[TMP0:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP1:%.*]] = call [[TMP0]] @continuations.getSystemData.s_s() ; LOWERRAYTRACINGPIPELINE-NEXT: store [[TMP0]] [[TMP1]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 @@ -451,29 +358,71 @@ attributes #5 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP5:%.*]] = bitcast ptr [[TMP4]] to ptr ; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr [[TMP5]]) #[[ATTR1:[0-9]+]] ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP4]], i32 0, i32 0 -; LOWERRAYTRACINGPIPELINE-NEXT: store <4 x float> zeroinitializer, ptr [[TMP6]], align 4, !tbaa [[TBAA23:![0-9]+]] +; LOWERRAYTRACINGPIPELINE-NEXT: store <4 x float> zeroinitializer, ptr [[TMP6]], align 4, !tbaa [[TBAA22:![0-9]+]] ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP7:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP2]]) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP8:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP7]], [[DX_TYPES_RESOURCEPROPERTIES:%.*]] { i32 16, i32 0 }) ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP9:%.*]] = call i64 @amd.dx.getAccelStructAddr([[DX_TYPES_HANDLE]] [[TMP8]]) -; LOWERRAYTRACINGPIPELINE-NEXT: call void @_cont_TraceRay.struct.RayPayload.attr_max_32_bytes(ptr [[SYSTEM_DATA_ALLOCA]], i64 [[TMP9]], i32 16, i32 -1, i32 0, i32 1, i32 0, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0x3F50624DE0000000, float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 1.000000e+04, ptr [[TMP4]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = load <4 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA23]] -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() -; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT:%.*]] = extractelement <3 x i32> [[TMP11]], i8 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() -; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT1:%.*]] = extractelement <3 x i32> [[TMP12]], i8 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP3]]) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP13]], [[DX_TYPES_RESOURCEPROPERTIES]] { i32 4098, i32 1033 }) -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = extractelement <4 x float> [[TMP10]], i64 0 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = extractelement <4 x float> [[TMP10]], i64 1 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = extractelement <4 x float> [[TMP10]], i64 2 -; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = extractelement <4 x float> [[TMP10]], i64 3 -; LOWERRAYTRACINGPIPELINE-NEXT: call void @dx.op.textureStore.f32(i32 67, [[DX_TYPES_HANDLE]] [[TMP14]], i32 [[EXTRACT]], i32 [[EXTRACT1]], i32 undef, float [[TMP15]], float [[TMP16]], float [[TMP17]], float [[TMP18]], i8 15) +; LOWERRAYTRACINGPIPELINE-NEXT: [[DIS_DATA_I:%.*]] = load [[TMP0]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[SYS_DATA_I:%.*]] = insertvalue [[TMP2]] undef, [[TMP0]] [[DIS_DATA_I]], 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA_I:%.*]] = insertvalue [[TMP1]] undef, [[TMP2]] [[SYS_DATA_I]], 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[ADDR_I:%.*]] = call i64 @_AmdGetResumePointAddr() #[[ATTR3:[0-9]+]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[TRAV_DATA2_I:%.*]] = insertvalue [[TMP1]] [[TRAV_DATA_I]], i64 [[ADDR_I]], 5 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP4]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[TMP10]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[TMP11]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP13]], ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[TMP10]], i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[TMP14]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP16]], ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP17:%.*]] = getelementptr i32, ptr [[TMP14]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP18]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[TMP14]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP19]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP20]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_0_CALLER_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP21:%.*]] = call ptr inttoptr (i64 4 to ptr)([[TMP1]] [[TRAV_DATA2_I]]), !continuation.registercount [[META25:![0-9]+]], !continuation.returnedRegistercount !25 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP22:%.*]] = call [[TMP0]] @await.(ptr [[TMP21]]) +; LOWERRAYTRACINGPIPELINE-NEXT: store [[STRUCT_RAYPAYLOAD]] poison, ptr [[TMP4]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP23:%.*]] = getelementptr inbounds [[STRUCT_RAYPAYLOAD]], ptr [[TMP4]], i32 0, i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP24:%.*]] = getelementptr i32, ptr [[TMP23]], i32 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr [[TMP24]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP26:%.*]] = load i32, ptr @PAYLOAD, align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP26]], ptr [[TMP25]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP27:%.*]] = getelementptr i32, ptr [[TMP23]], i32 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr [[TMP27]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP29:%.*]] = load i32, ptr getelementptr inbounds ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT:%.*]], ptr @PAYLOAD, i32 0, i32 0, i32 7), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP29]], ptr [[TMP28]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP30:%.*]] = getelementptr i32, ptr [[TMP27]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP31:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 8), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP31]], ptr [[TMP30]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr [[TMP27]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP33:%.*]] = load i32, ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP33]], ptr [[TMP32]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: store [[TMP0]] [[TMP22]], ptr [[SYSTEM_DATA_ALLOCA]], align 4 +; LOWERRAYTRACINGPIPELINE-NEXT: [[LOCAL_ROOT_INDEX1:%.*]] = call i32 @_cont_GetLocalRootIndex(ptr [[SYSTEM_DATA_ALLOCA]]) +; LOWERRAYTRACINGPIPELINE-NEXT: call void @amd.dx.setLocalRootIndex(i32 [[LOCAL_ROOT_INDEX1]]) +; LOWERRAYTRACINGPIPELINE-NEXT: br label [[DOTSPLIT:%.*]] +; LOWERRAYTRACINGPIPELINE: .split: +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP34:%.*]] = load <4 x float>, ptr [[TMP6]], align 4, !tbaa [[TBAA22]] +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP35:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() +; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT:%.*]] = extractelement <3 x i32> [[TMP35]], i8 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP36:%.*]] = call <3 x i32> @lgc.rt.dispatch.rays.index() +; LOWERRAYTRACINGPIPELINE-NEXT: [[EXTRACT1:%.*]] = extractelement <3 x i32> [[TMP36]], i8 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP37:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.createHandleForLib.dx.types.Handle(i32 160, [[DX_TYPES_HANDLE]] [[TMP3]]) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP38:%.*]] = call [[DX_TYPES_HANDLE]] @dx.op.annotateHandle(i32 216, [[DX_TYPES_HANDLE]] [[TMP37]], [[DX_TYPES_RESOURCEPROPERTIES]] { i32 4098, i32 1033 }) +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP39:%.*]] = extractelement <4 x float> [[TMP34]], i64 0 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP40:%.*]] = extractelement <4 x float> [[TMP34]], i64 1 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP41:%.*]] = extractelement <4 x float> [[TMP34]], i64 2 +; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP42:%.*]] = extractelement <4 x float> [[TMP34]], i64 3 +; LOWERRAYTRACINGPIPELINE-NEXT: call void @dx.op.textureStore.f32(i32 67, [[DX_TYPES_HANDLE]] [[TMP38]], i32 [[EXTRACT]], i32 [[EXTRACT1]], i32 undef, float [[TMP39]], float [[TMP40]], float [[TMP41]], float [[TMP42]], i8 15) ; LOWERRAYTRACINGPIPELINE-NEXT: call void @llvm.lifetime.end.p0(i64 16, ptr [[TMP5]]) #[[ATTR1]] -; LOWERRAYTRACINGPIPELINE-NEXT: ret void +; LOWERRAYTRACINGPIPELINE-NEXT: ret void, !continuation.registercount [[META18:![0-9]+]] ; ; ; LOWERRAYTRACINGPIPELINE-LABEL: define %0 @MyClosestHit( -; LOWERRAYTRACINGPIPELINE-SAME: [[TMP2:%.*]] [[TMP0:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage !26 !continuation.registercount !20 !continuation !27 { +; LOWERRAYTRACINGPIPELINE-SAME: [[TMP2:%.*]] [[TMP0:%.*]]) #[[ATTR2]] !lgc.rt.shaderstage [[META26:![0-9]+]] !continuation.registercount [[META25]] !continuation [[META27:![0-9]+]] { ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP2]] = alloca [[STRUCT_BUILTINTRIANGLEINTERSECTIONATTRIBUTES:%.*]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[SYSTEM_DATA_ALLOCA:%.*]] = alloca [[TMP2]], align 8 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP3:%.*]] = alloca [[STRUCT_RAYPAYLOAD:%.*]], align 8 @@ -539,5 +488,5 @@ attributes #5 = { nocallback nofree nosync nounwind willreturn memory(argmem: re ; LOWERRAYTRACINGPIPELINE-NEXT: store i32 [[TMP45]], ptr getelementptr ([[STRUCT_RAYPAYLOAD_ATTR_MAX_8_I32S_LAYOUT_5_CLOSESTHIT_OUT]], ptr @PAYLOAD, i32 0, i32 0, i64 9), align 4 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP46:%.*]] = getelementptr inbounds [[TMP2]], ptr [[SYSTEM_DATA_ALLOCA]], i32 0, i32 0 ; LOWERRAYTRACINGPIPELINE-NEXT: [[TMP47:%.*]] = load [[TMP0]], ptr [[TMP46]], align 4 -; LOWERRAYTRACINGPIPELINE-NEXT: ret [[TMP0]] [[TMP47]], !continuation.registercount !20 +; LOWERRAYTRACINGPIPELINE-NEXT: ret [[TMP0]] [[TMP47]], !continuation.registercount [[META25]] ; diff --git a/shared/continuations/test/intrinsics/discard-values.ll b/shared/continuations/test/intrinsics/discard-values.ll index 399fbeef80..421489f008 100644 --- a/shared/continuations/test/intrinsics/discard-values.ll +++ b/shared/continuations/test/intrinsics/discard-values.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 -; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint' -S %s 2>%t.stderr | FileCheck %s +; RUN: opt --verify-each -passes='dxil-cont-intrinsic-prepare,lint' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr %struct.AnyHitData = type { float, i32 } diff --git a/shared/continuations/test/intrinsics/get-func-addr.ll b/shared/continuations/test/intrinsics/get-func-addr.ll index 0ad7e648b4..496e1eaf91 100644 --- a/shared/continuations/test/intrinsics/get-func-addr.ll +++ b/shared/continuations/test/intrinsics/get-func-addr.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 -; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint' -S %s 2>%t.stderr | FileCheck %s +; RUN: opt --verify-each -passes='dxil-cont-lgc-rt-op-converter,lint,lower-raytracing-pipeline,lint' -S %s 2> %t.stderr | FileCheck %s ; RUN: count 0 < %t.stderr %struct.DispatchSystemData = type { i32 } diff --git a/shared/continuations/test/lgccps/alloca-select.ll b/shared/continuations/test/lgccps/alloca-select.ll index 32b00e3f1f..a935c900c1 100644 --- a/shared/continuations/test/lgccps/alloca-select.ll +++ b/shared/continuations/test/lgccps/alloca-select.ll @@ -27,8 +27,8 @@ declare void @lgc.cps.jump(...) ; CHECK-SAME: ({} [[STATE:%.*]], i32 [[RCR:%.*]], float [[ARG:%.*]], i32 [[ARG1:%.*]]) !lgc.cps !0 !continuation !1 { ; CHECK-NEXT: AllocaSpillBB: ; CHECK-NEXT: [[TMP0:%.*]] = call ptr addrspace(32) @lgc.cps.alloc(i32 20) -; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP0]], i32 0, i32 0 -; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 1 +; CHECK-NEXT: [[A1:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP0]], i32 0, i32 0 +; CHECK-NEXT: [[A2:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 1 ; CHECK-NEXT: [[ARG1_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 4 ; CHECK-NEXT: store i32 [[ARG1]], ptr addrspace(32) [[ARG1_SPILL_ADDR]], align 4 ; CHECK-NEXT: [[ARG_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 3 @@ -36,22 +36,22 @@ declare void @lgc.cps.jump(...) ; CHECK-NEXT: [[RCR_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 2 ; CHECK-NEXT: store i32 [[RCR]], ptr addrspace(32) [[RCR_SPILL_ADDR]], align 4 ; CHECK-NEXT: [[COND:%.*]] = icmp ult i32 [[ARG1]], 0 -; CHECK-NEXT: [[P:%.*]] = select i1 [[COND]], ptr addrspace(32) [[TMP1]], ptr addrspace(32) [[TMP2]] +; CHECK-NEXT: [[P:%.*]] = select i1 [[COND]], ptr addrspace(32) [[A1]], ptr addrspace(32) [[A2]] ; CHECK-NEXT: store i32 111, ptr addrspace(32) [[P]], align 4 ; CHECK-NEXT: [[T0:%.*]] = fadd float [[ARG]], 1.000000e+00 ; CHECK-NEXT: [[CR:%.*]] = call i32 @lgc.cps.as.continuation.reference(ptr @callee) -; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i32 [[CR]] to ptr -; CHECK-NEXT: [[TMP4:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.0) -; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR]], i32 2, {} poison, i32 [[TMP4]], float [[T0]]) +; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i32 [[CR]] to ptr +; CHECK-NEXT: [[TMP2:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.0) +; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR]], i32 2, {} poison, i32 [[TMP2]], float [[T0]]) ; CHECK-NEXT: unreachable ; ; -; CHECK-LABEL: define void @test.resume.0 +; CHECK-LABEL: define dso_local void @test.resume.0 ; CHECK-SAME: ({} [[TMP0:%.*]], i32 [[TMP1:%.*]], float [[TMP2:%.*]]) !lgc.cps !0 !continuation !1 { ; CHECK-NEXT: entryresume.0: ; CHECK-NEXT: [[TMP3:%.*]] = call ptr addrspace(32) @lgc.cps.peek(i32 20) -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 1 +; CHECK-NEXT: [[A1:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 0 +; CHECK-NEXT: [[A2:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 1 ; CHECK-NEXT: [[ARG1_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 4 ; CHECK-NEXT: [[ARG1_RELOAD:%.*]] = load i32, ptr addrspace(32) [[ARG1_RELOAD_ADDR]], align 4 ; CHECK-NEXT: [[ARG_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 3 @@ -59,7 +59,7 @@ declare void @lgc.cps.jump(...) ; CHECK-NEXT: [[RCR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 2 ; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[RCR_RELOAD_ADDR]], align 4 ; CHECK-NEXT: [[COND2:%.*]] = icmp ult i32 [[ARG1_RELOAD]], 0 -; CHECK-NEXT: [[P1:%.*]] = select i1 [[COND2]], ptr addrspace(32) [[TMP4]], ptr addrspace(32) [[TMP5]] +; CHECK-NEXT: [[P1:%.*]] = select i1 [[COND2]], ptr addrspace(32) [[A1]], ptr addrspace(32) [[A2]] ; CHECK-NEXT: [[TMP:%.*]] = fmul float [[TMP2]], [[ARG_RELOAD]] ; CHECK-NEXT: [[V111:%.*]] = load float, ptr addrspace(32) [[P1]], align 4 ; CHECK-NEXT: [[RETURNVALUE:%.*]] = fmul float [[TMP]], [[V111]] diff --git a/shared/continuations/test/lgccps/await-if-else.ll b/shared/continuations/test/lgccps/await-if-else.ll index e6380db3bd..370943b73b 100644 --- a/shared/continuations/test/lgccps/await-if-else.ll +++ b/shared/continuations/test/lgccps/await-if-else.ll @@ -34,49 +34,49 @@ declare void @lgc.cps.jump(...) ; CHECK-SAME: {} [[STATE:%.*]], i32 [[RCR:%.*]], float [[ARG:%.*]]) !lgc.cps !0 !continuation !1 { ; CHECK-NEXT: AllocaSpillBB: ; CHECK-NEXT: [[TMP0:%.*]] = call ptr addrspace(32) @lgc.cps.alloc(i32 8) -; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP0]], i32 0, i32 1 -; CHECK-NEXT: store float [[ARG]], ptr addrspace(32) [[TMP1]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 0 -; CHECK-NEXT: store i32 [[RCR]], ptr addrspace(32) [[TMP2]], align 4 +; CHECK-NEXT: [[ARG_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP0]], i32 0, i32 1 +; CHECK-NEXT: store float [[ARG]], ptr addrspace(32) [[ARG_SPILL_ADDR]], align 4 +; CHECK-NEXT: [[RCR_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 0 +; CHECK-NEXT: store i32 [[RCR]], ptr addrspace(32) [[RCR_SPILL_ADDR]], align 4 ; CHECK-NEXT: [[T0:%.*]] = fadd float [[ARG]], 1.000000e+00 ; CHECK-NEXT: [[CR:%.*]] = call i32 @lgc.cps.as.continuation.reference(ptr @callee) ; CHECK-NEXT: [[CR2:%.*]] = call i32 @lgc.cps.as.continuation.reference(ptr @callee2) ; CHECK-NEXT: [[COND:%.*]] = fcmp olt float [[T0]], 1.000000e+00 ; CHECK-NEXT: br i1 [[COND]], label [[BB1:%.*]], label [[BB2:%.*]] ; CHECK: bb1: -; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i32 [[CR]] to ptr -; CHECK-NEXT: [[TMP4:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.0) -; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR]], i32 2, {} poison, i32 [[TMP4]], float [[ARG]]) +; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i32 [[CR]] to ptr +; CHECK-NEXT: [[TMP2:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.0) +; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR]], i32 2, {} poison, i32 [[TMP2]], float [[ARG]]) ; CHECK-NEXT: unreachable ; CHECK: bb2: -; CHECK-NEXT: [[TMP5:%.*]] = inttoptr i32 [[CR2]] to ptr -; CHECK-NEXT: [[TMP6:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.1) -; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR2]], i32 2, {} poison, i32 [[TMP6]], float [[T0]]) +; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i32 [[CR2]] to ptr +; CHECK-NEXT: [[TMP4:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.1) +; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR2]], i32 2, {} poison, i32 [[TMP4]], float [[T0]]) ; CHECK-NEXT: unreachable ; ; -; CHECK-LABEL: define void @test.resume.0( +; CHECK-LABEL: define dso_local void @test.resume.0( ; CHECK-SAME: {} [[TMP0:%.*]], i32 [[TMP1:%.*]], float [[TMP2:%.*]]) !lgc.cps !0 !continuation !1 { ; CHECK-NEXT: entryresume.0: ; CHECK-NEXT: [[TMP3:%.*]] = call ptr addrspace(32) @lgc.cps.peek(i32 8) -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 1 -; CHECK-NEXT: [[ARG_RELOAD:%.*]] = load float, ptr addrspace(32) [[TMP4]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 0 -; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[TMP5]], align 4 +; CHECK-NEXT: [[ARG_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 1 +; CHECK-NEXT: [[ARG_RELOAD:%.*]] = load float, ptr addrspace(32) [[ARG_RELOAD_ADDR]], align 4 +; CHECK-NEXT: [[RCR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 0 +; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[RCR_RELOAD_ADDR]], align 4 ; CHECK-NEXT: [[RETURNVALUE:%.*]] = fmul float [[TMP2]], [[ARG_RELOAD]] ; CHECK-NEXT: call void @lgc.cps.free(i32 8) ; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[RCR_RELOAD]], i32 2, {} poison, i32 poison, float [[RETURNVALUE]]) ; CHECK-NEXT: unreachable ; ; -; CHECK-LABEL: define void @test.resume.1( +; CHECK-LABEL: define dso_local void @test.resume.1( ; CHECK-SAME: {} [[TMP0:%.*]], i32 [[TMP1:%.*]], float [[TMP2:%.*]]) !lgc.cps !0 !continuation !1 { ; CHECK-NEXT: entryresume.1: ; CHECK-NEXT: [[TMP3:%.*]] = call ptr addrspace(32) @lgc.cps.peek(i32 8) -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 1 -; CHECK-NEXT: [[ARG_RELOAD:%.*]] = load float, ptr addrspace(32) [[TMP4]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 0 -; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[TMP5]], align 4 +; CHECK-NEXT: [[ARG_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 1 +; CHECK-NEXT: [[ARG_RELOAD:%.*]] = load float, ptr addrspace(32) [[ARG_RELOAD_ADDR]], align 4 +; CHECK-NEXT: [[RCR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 0 +; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[RCR_RELOAD_ADDR]], align 4 ; CHECK-NEXT: [[RETURNVALUE:%.*]] = fmul float [[TMP2]], [[ARG_RELOAD]] ; CHECK-NEXT: call void @lgc.cps.free(i32 8) ; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[RCR_RELOAD]], i32 2, {} poison, i32 poison, float [[RETURNVALUE]]) diff --git a/shared/continuations/test/lgccps/await-if.ll b/shared/continuations/test/lgccps/await-if.ll index 6f5f78615c..1076ebe0e8 100644 --- a/shared/continuations/test/lgccps/await-if.ll +++ b/shared/continuations/test/lgccps/await-if.ll @@ -30,39 +30,39 @@ declare void @lgc.cps.jump(...) ; CHECK-SAME: {} [[STATE:%.*]], i32 [[RCR:%.*]], float [[ARG:%.*]]) !lgc.cps !0 !continuation !1 { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = call ptr addrspace(32) @lgc.cps.alloc(i32 8) -; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP0]], i32 0, i32 1 -; CHECK-NEXT: store float [[ARG]], ptr addrspace(32) [[TMP1]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 0 -; CHECK-NEXT: store i32 [[RCR]], ptr addrspace(32) [[TMP2]], align 4 +; CHECK-NEXT: [[ARG_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP0]], i32 0, i32 1 +; CHECK-NEXT: store float [[ARG]], ptr addrspace(32) [[ARG_SPILL_ADDR]], align 4 +; CHECK-NEXT: [[RCR_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 0 +; CHECK-NEXT: store i32 [[RCR]], ptr addrspace(32) [[RCR_SPILL_ADDR]], align 4 ; CHECK-NEXT: [[T0:%.*]] = fadd float [[ARG]], 1.000000e+00 ; CHECK-NEXT: [[CR:%.*]] = call i32 @lgc.cps.as.continuation.reference(ptr @callee) ; CHECK-NEXT: [[COND:%.*]] = fcmp olt float [[T0]], 1.000000e+00 ; CHECK-NEXT: br i1 [[COND]], label [[BB1:%.*]], label [[BB2:%.*]] ; CHECK: bb1: -; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i32 [[CR]] to ptr -; CHECK-NEXT: [[TMP4:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.0) -; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR]], i32 2, {} poison, i32 [[TMP4]], float [[ARG]]) +; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i32 [[CR]] to ptr +; CHECK-NEXT: [[TMP2:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.0) +; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR]], i32 2, {} poison, i32 [[TMP2]], float [[ARG]]) ; CHECK-NEXT: unreachable ; CHECK: bb2: ; CHECK-NEXT: [[T0_BB2:%.*]] = phi float [ [[T0]], [[ENTRY:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 1 -; CHECK-NEXT: [[ARG_RELOAD:%.*]] = load float, ptr addrspace(32) [[TMP5]], align 4 -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 0 -; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[TMP6]], align 4 +; CHECK-NEXT: [[ARG_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 1 +; CHECK-NEXT: [[ARG_RELOAD:%.*]] = load float, ptr addrspace(32) [[ARG_RELOAD_ADDR]], align 4 +; CHECK-NEXT: [[RCR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 0 +; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[RCR_RELOAD_ADDR]], align 4 ; CHECK-NEXT: [[RETURNVALUE:%.*]] = fmul float [[T0_BB2]], [[ARG_RELOAD]] ; CHECK-NEXT: call void @lgc.cps.free(i32 8) ; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[RCR_RELOAD]], i32 2, {} poison, i32 poison, float [[RETURNVALUE]]) ; CHECK-NEXT: unreachable ; ; -; CHECK-LABEL: define void @test.resume.0( +; CHECK-LABEL: define dso_local void @test.resume.0( ; CHECK-SAME: {} [[TMP0:%.*]], i32 [[TMP1:%.*]], float [[TMP2:%.*]]) !lgc.cps !0 !continuation !1 { ; CHECK-NEXT: entryresume.0: ; CHECK-NEXT: [[TMP3:%.*]] = call ptr addrspace(32) @lgc.cps.peek(i32 8) -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 1 -; CHECK-NEXT: [[ARG_RELOAD:%.*]] = load float, ptr addrspace(32) [[TMP4]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 0 -; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[TMP5]], align 4 +; CHECK-NEXT: [[ARG_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 1 +; CHECK-NEXT: [[ARG_RELOAD:%.*]] = load float, ptr addrspace(32) [[ARG_RELOAD_ADDR]], align 4 +; CHECK-NEXT: [[RCR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 0 +; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[RCR_RELOAD_ADDR]], align 4 ; CHECK-NEXT: [[RETURNVALUE:%.*]] = fmul float [[TMP2]], [[ARG_RELOAD]] ; CHECK-NEXT: call void @lgc.cps.free(i32 8) ; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[RCR_RELOAD]], i32 2, {} poison, i32 poison, float [[RETURNVALUE]]) diff --git a/shared/continuations/test/lgccps/await-in-loop.ll b/shared/continuations/test/lgccps/await-in-loop.ll index 76e2f1d75e..ead48b312d 100644 --- a/shared/continuations/test/lgccps/await-in-loop.ll +++ b/shared/continuations/test/lgccps/await-in-loop.ll @@ -32,52 +32,52 @@ declare void @lgc.cps.jump(...) ; CHECK-SAME: {} [[STATE:%.*]], i32 [[RCR:%.*]], float [[ARG:%.*]], float [[ARG2:%.*]]) !lgc.cps !0 !continuation !1 { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = call ptr addrspace(32) @lgc.cps.alloc(i32 20) -; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP0]], i32 0, i32 2 -; CHECK-NEXT: store float [[ARG2]], ptr addrspace(32) [[TMP1]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 1 -; CHECK-NEXT: store float [[ARG]], ptr addrspace(32) [[TMP2]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 0 -; CHECK-NEXT: store i32 [[RCR]], ptr addrspace(32) [[TMP3]], align 4 +; CHECK-NEXT: [[ARG2_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP0]], i32 0, i32 2 +; CHECK-NEXT: store float [[ARG2]], ptr addrspace(32) [[ARG2_SPILL_ADDR]], align 4 +; CHECK-NEXT: [[ARG_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 1 +; CHECK-NEXT: store float [[ARG]], ptr addrspace(32) [[ARG_SPILL_ADDR]], align 4 +; CHECK-NEXT: [[RCR_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 0 +; CHECK-NEXT: store i32 [[RCR]], ptr addrspace(32) [[RCR_SPILL_ADDR]], align 4 ; CHECK-NEXT: [[T0:%.*]] = fadd float [[ARG]], 1.000000e+00 ; CHECK-NEXT: [[CR:%.*]] = call i32 @lgc.cps.as.continuation.reference(ptr @callee) -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 3 -; CHECK-NEXT: store i32 [[CR]], ptr addrspace(32) [[TMP4]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 4 -; CHECK-NEXT: store i32 0, ptr addrspace(32) [[TMP5]], align 4 -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 3 -; CHECK-NEXT: [[CR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[TMP6]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = inttoptr i32 [[CR_RELOAD]] to ptr -; CHECK-NEXT: [[TMP8:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.0) -; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR_RELOAD]], i32 2, {} poison, i32 [[TMP8]], i32 0) +; CHECK-NEXT: [[CR_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 3 +; CHECK-NEXT: store i32 [[CR]], ptr addrspace(32) [[CR_SPILL_ADDR]], align 4 +; CHECK-NEXT: [[IND_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 4 +; CHECK-NEXT: store i32 0, ptr addrspace(32) [[IND_SPILL_ADDR]], align 4 +; CHECK-NEXT: [[CR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 3 +; CHECK-NEXT: [[CR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[CR_RELOAD_ADDR]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i32 [[CR_RELOAD]] to ptr +; CHECK-NEXT: [[TMP2:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.0) +; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR_RELOAD]], i32 2, {} poison, i32 [[TMP2]], i32 0) ; CHECK-NEXT: unreachable ; ; -; CHECK-LABEL: define void @test.resume.0( +; CHECK-LABEL: define dso_local void @test.resume.0( ; CHECK-SAME: {} [[TMP0:%.*]], i32 [[TMP1:%.*]], float [[TMP2:%.*]]) !lgc.cps !0 !continuation !1 { ; CHECK-NEXT: entryresume.0: ; CHECK-NEXT: [[TMP3:%.*]] = call ptr addrspace(32) @lgc.cps.peek(i32 20) -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 4 -; CHECK-NEXT: [[IND_RELOAD:%.*]] = load i32, ptr addrspace(32) [[TMP4]], align 4 +; CHECK-NEXT: [[IND_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 4 +; CHECK-NEXT: [[IND_RELOAD:%.*]] = load i32, ptr addrspace(32) [[IND_RELOAD_ADDR]], align 4 ; CHECK-NEXT: [[INC:%.*]] = add i32 [[IND_RELOAD]], 1 ; CHECK-NEXT: [[COND:%.*]] = fcmp olt float [[TMP2]], 5.000000e+00 ; CHECK-NEXT: br i1 [[COND]], label [[LOOP_FROM_AFTERCOROSUSPEND:%.*]], label [[END:%.*]] ; CHECK: loop.from.AfterCoroSuspend: ; CHECK-NEXT: [[INC_LOOP:%.*]] = phi i32 [ [[INC]], [[ENTRYRESUME_0:%.*]] ] -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 4 -; CHECK-NEXT: store i32 [[INC_LOOP]], ptr addrspace(32) [[TMP5]], align 4 -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 3 -; CHECK-NEXT: [[CR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[TMP6]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = inttoptr i32 [[CR_RELOAD]] to ptr -; CHECK-NEXT: [[TMP8:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.0) -; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR_RELOAD]], i32 2, {} poison, i32 [[TMP8]], i32 [[INC_LOOP]]) +; CHECK-NEXT: [[IND_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 4 +; CHECK-NEXT: store i32 [[INC_LOOP]], ptr addrspace(32) [[IND_SPILL_ADDR]], align 4 +; CHECK-NEXT: [[CR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 3 +; CHECK-NEXT: [[CR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[CR_RELOAD_ADDR]], align 4 +; CHECK-NEXT: [[TMP4:%.*]] = inttoptr i32 [[CR_RELOAD]] to ptr +; CHECK-NEXT: [[TMP5:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.0) +; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR_RELOAD]], i32 2, {} poison, i32 [[TMP5]], i32 [[INC_LOOP]]) ; CHECK-NEXT: unreachable ; CHECK: end: -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 2 -; CHECK-NEXT: [[ARG2_RELOAD:%.*]] = load float, ptr addrspace(32) [[TMP9]], align 4 -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 1 -; CHECK-NEXT: [[ARG_RELOAD:%.*]] = load float, ptr addrspace(32) [[TMP10]], align 4 -; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 0 -; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[TMP11]], align 4 +; CHECK-NEXT: [[ARG2_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 2 +; CHECK-NEXT: [[ARG2_RELOAD:%.*]] = load float, ptr addrspace(32) [[ARG2_RELOAD_ADDR]], align 4 +; CHECK-NEXT: [[ARG_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 1 +; CHECK-NEXT: [[ARG_RELOAD:%.*]] = load float, ptr addrspace(32) [[ARG_RELOAD_ADDR]], align 4 +; CHECK-NEXT: [[RCR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 0 +; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[RCR_RELOAD_ADDR]], align 4 ; CHECK-NEXT: [[T2:%.*]] = fmul float [[TMP2]], [[ARG_RELOAD]] ; CHECK-NEXT: [[RETURNVALUE:%.*]] = fadd float [[T2]], [[ARG2_RELOAD]] ; CHECK-NEXT: call void @lgc.cps.free(i32 20) diff --git a/shared/continuations/test/lgccps/entry-point-with-cps.ll b/shared/continuations/test/lgccps/entry-point-with-cps.ll index 2d0ea41865..a9b678ebb1 100644 --- a/shared/continuations/test/lgccps/entry-point-with-cps.ll +++ b/shared/continuations/test/lgccps/entry-point-with-cps.ll @@ -80,7 +80,7 @@ declare void @lgc.cps.jump(...) ; CHECK-NEXT: unreachable ; ; -; CHECK-LABEL: define void @raygen.resume.0( +; CHECK-LABEL: define dso_local void @raygen.resume.0( ; CHECK-SAME: {} [[TMP0:%.*]], i32 [[TMP1:%.*]], [2 x i32] [[TMP2:%.*]]) !lgc.shaderstage !0 !lgc.cps !1 !continuation !2 { ; CHECK-NEXT: entryresume.0: ; CHECK-NEXT: [[PUSHCONST3:%.*]] = call ptr addrspace(4) @lgc.user.data(i32 0) @@ -94,24 +94,24 @@ declare void @lgc.cps.jump(...) ; CHECK-SAME: {} [[STATE:%.*]], i32 [[RCR:%.*]], i32 [[X:%.*]]) !lgc.shaderstage !0 !lgc.cps !3 !continuation !4 { ; CHECK-NEXT: AllocaSpillBB: ; CHECK-NEXT: [[TMP0:%.*]] = call ptr addrspace(32) @lgc.cps.alloc(i32 8) -; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[CHS_FRAME:%.*]], ptr addrspace(32) [[TMP0]], i32 0, i32 0 -; CHECK-NEXT: store i32 [[RCR]], ptr addrspace(32) [[TMP1]], align 4 +; CHECK-NEXT: [[RCR_SPILL_ADDR:%.*]] = getelementptr inbounds [[CHS_FRAME:%.*]], ptr addrspace(32) [[TMP0]], i32 0, i32 0 +; CHECK-NEXT: store i32 [[RCR]], ptr addrspace(32) [[RCR_SPILL_ADDR]], align 4 ; CHECK-NEXT: [[PUSHCONST:%.*]] = call ptr addrspace(4) @lgc.user.data(i32 24) ; CHECK-NEXT: [[FN:%.*]] = load ptr, ptr addrspace(4) [[PUSHCONST]], align 8 ; CHECK-NEXT: [[CR_0:%.*]] = ptrtoint ptr [[FN]] to i32 ; CHECK-NEXT: [[CR_1:%.*]] = or i32 [[CR_0]], 1 -; CHECK-NEXT: [[TMP2:%.*]] = inttoptr i32 [[CR_1]] to ptr -; CHECK-NEXT: [[TMP3:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @chs.resume.0) -; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR_1]], i32 2, {} poison, i32 [[TMP3]], i32 [[X]]) +; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i32 [[CR_1]] to ptr +; CHECK-NEXT: [[TMP2:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @chs.resume.0) +; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR_1]], i32 2, {} poison, i32 [[TMP2]], i32 [[X]]) ; CHECK-NEXT: unreachable ; ; -; CHECK-LABEL: define void @chs.resume.0( +; CHECK-LABEL: define dso_local void @chs.resume.0( ; CHECK-SAME: {} [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]]) !lgc.shaderstage !0 !lgc.cps !3 !continuation !4 { ; CHECK-NEXT: entryresume.0: ; CHECK-NEXT: [[TMP3:%.*]] = call ptr addrspace(32) @lgc.cps.peek(i32 8) -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[CHS_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 0 -; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[TMP4]], align 4 +; CHECK-NEXT: [[RCR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[CHS_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 0 +; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[RCR_RELOAD_ADDR]], align 4 ; CHECK-NEXT: call void @lgc.cps.free(i32 8) ; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[RCR_RELOAD]], i32 5, i32 [[TMP2]]) ; CHECK-NEXT: unreachable diff --git a/shared/continuations/test/lgccps/multiple-await.ll b/shared/continuations/test/lgccps/multiple-await.ll index 3500f49d40..8e51ba6344 100644 --- a/shared/continuations/test/lgccps/multiple-await.ll +++ b/shared/continuations/test/lgccps/multiple-await.ll @@ -25,42 +25,42 @@ declare void @lgc.cps.jump(...) ; CHECK-SAME: {} [[STATE:%.*]], i32 [[RCR:%.*]], float [[ARG:%.*]], float [[ARG2:%.*]]) !lgc.cps !0 !continuation !1 { ; CHECK-NEXT: AllocaSpillBB: ; CHECK-NEXT: [[TMP0:%.*]] = call ptr addrspace(32) @lgc.cps.alloc(i32 12) -; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP0]], i32 0, i32 2 -; CHECK-NEXT: store float [[ARG2]], ptr addrspace(32) [[TMP1]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 1 -; CHECK-NEXT: store float [[ARG]], ptr addrspace(32) [[TMP2]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 0 -; CHECK-NEXT: store i32 [[RCR]], ptr addrspace(32) [[TMP3]], align 4 +; CHECK-NEXT: [[ARG2_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP0]], i32 0, i32 2 +; CHECK-NEXT: store float [[ARG2]], ptr addrspace(32) [[ARG2_SPILL_ADDR]], align 4 +; CHECK-NEXT: [[ARG_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 1 +; CHECK-NEXT: store float [[ARG]], ptr addrspace(32) [[ARG_SPILL_ADDR]], align 4 +; CHECK-NEXT: [[RCR_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 0 +; CHECK-NEXT: store i32 [[RCR]], ptr addrspace(32) [[RCR_SPILL_ADDR]], align 4 ; CHECK-NEXT: [[T0:%.*]] = fadd float [[ARG]], 1.000000e+00 ; CHECK-NEXT: [[CR:%.*]] = call i32 @lgc.cps.as.continuation.reference(ptr @callee) -; CHECK-NEXT: [[TMP4:%.*]] = inttoptr i32 [[CR]] to ptr -; CHECK-NEXT: [[TMP5:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.0) -; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR]], i32 2, {} poison, i32 [[TMP5]], float [[T0]]) +; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i32 [[CR]] to ptr +; CHECK-NEXT: [[TMP2:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.0) +; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR]], i32 2, {} poison, i32 [[TMP2]], float [[T0]]) ; CHECK-NEXT: unreachable ; ; -; CHECK-LABEL: define void @test.resume.0( +; CHECK-LABEL: define dso_local void @test.resume.0( ; CHECK-SAME: {} [[TMP0:%.*]], i32 [[TMP1:%.*]], float [[TMP2:%.*]]) !lgc.cps !0 !continuation !1 { ; CHECK-NEXT: entryresume.0: ; CHECK-NEXT: [[TMP3:%.*]] = call ptr addrspace(32) @lgc.cps.peek(i32 12) -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 1 -; CHECK-NEXT: [[ARG_RELOAD:%.*]] = load float, ptr addrspace(32) [[TMP4]], align 4 +; CHECK-NEXT: [[ARG_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 1 +; CHECK-NEXT: [[ARG_RELOAD:%.*]] = load float, ptr addrspace(32) [[ARG_RELOAD_ADDR]], align 4 ; CHECK-NEXT: [[T2:%.*]] = fmul float [[TMP2]], [[ARG_RELOAD]] ; CHECK-NEXT: [[CR2:%.*]] = call i32 @lgc.cps.as.continuation.reference(ptr @callee2) -; CHECK-NEXT: [[TMP5:%.*]] = inttoptr i32 [[CR2]] to ptr -; CHECK-NEXT: [[TMP6:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.1) -; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR2]], i32 2, {} poison, i32 [[TMP6]], float [[T2]]) +; CHECK-NEXT: [[TMP4:%.*]] = inttoptr i32 [[CR2]] to ptr +; CHECK-NEXT: [[TMP5:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.1) +; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR2]], i32 2, {} poison, i32 [[TMP5]], float [[T2]]) ; CHECK-NEXT: unreachable ; ; -; CHECK-LABEL: define void @test.resume.1( +; CHECK-LABEL: define dso_local void @test.resume.1( ; CHECK-SAME: {} [[TMP0:%.*]], i32 [[TMP1:%.*]], float [[TMP2:%.*]]) !lgc.cps !0 !continuation !1 { ; CHECK-NEXT: entryresume.1: ; CHECK-NEXT: [[TMP3:%.*]] = call ptr addrspace(32) @lgc.cps.peek(i32 12) -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 2 -; CHECK-NEXT: [[ARG2_RELOAD:%.*]] = load float, ptr addrspace(32) [[TMP4]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 0 -; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[TMP5]], align 4 +; CHECK-NEXT: [[ARG2_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 2 +; CHECK-NEXT: [[ARG2_RELOAD:%.*]] = load float, ptr addrspace(32) [[ARG2_RELOAD_ADDR]], align 4 +; CHECK-NEXT: [[RCR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 0 +; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[RCR_RELOAD_ADDR]], align 4 ; CHECK-NEXT: [[RETURNVALUE:%.*]] = fadd float [[TMP2]], [[ARG2_RELOAD]] ; CHECK-NEXT: call void @lgc.cps.free(i32 12) ; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[RCR_RELOAD]], i32 2, {} poison, i32 poison, float [[RETURNVALUE]]) diff --git a/shared/continuations/test/lgccps/simple-await-more-state.ll b/shared/continuations/test/lgccps/simple-await-more-state.ll index 7ed851fdcb..a5a11a395c 100644 --- a/shared/continuations/test/lgccps/simple-await-more-state.ll +++ b/shared/continuations/test/lgccps/simple-await-more-state.ll @@ -22,30 +22,30 @@ declare void @lgc.cps.jump(...) ; CHECK-SAME: {} [[STATE:%.*]], i32 [[RCR:%.*]], float [[ARG:%.*]], float [[ARG2:%.*]]) !lgc.cps !0 !continuation !1 { ; CHECK-NEXT: AllocaSpillBB: ; CHECK-NEXT: [[TMP0:%.*]] = call ptr addrspace(32) @lgc.cps.alloc(i32 12) -; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP0]], i32 0, i32 2 -; CHECK-NEXT: store float [[ARG2]], ptr addrspace(32) [[TMP1]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 1 -; CHECK-NEXT: store float [[ARG]], ptr addrspace(32) [[TMP2]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 0 -; CHECK-NEXT: store i32 [[RCR]], ptr addrspace(32) [[TMP3]], align 4 +; CHECK-NEXT: [[ARG2_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP0]], i32 0, i32 2 +; CHECK-NEXT: store float [[ARG2]], ptr addrspace(32) [[ARG2_SPILL_ADDR]], align 4 +; CHECK-NEXT: [[ARG_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 1 +; CHECK-NEXT: store float [[ARG]], ptr addrspace(32) [[ARG_SPILL_ADDR]], align 4 +; CHECK-NEXT: [[RCR_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 0 +; CHECK-NEXT: store i32 [[RCR]], ptr addrspace(32) [[RCR_SPILL_ADDR]], align 4 ; CHECK-NEXT: [[T0:%.*]] = fadd float [[ARG]], 1.000000e+00 ; CHECK-NEXT: [[CR:%.*]] = call i32 @lgc.cps.as.continuation.reference(ptr @callee) -; CHECK-NEXT: [[TMP4:%.*]] = inttoptr i32 [[CR]] to ptr -; CHECK-NEXT: [[TMP5:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.0) -; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR]], i32 2, {} poison, i32 [[TMP5]], float [[T0]]) +; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i32 [[CR]] to ptr +; CHECK-NEXT: [[TMP2:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.0) +; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR]], i32 2, {} poison, i32 [[TMP2]], float [[T0]]) ; CHECK-NEXT: unreachable ; ; -; CHECK-LABEL: define void @test.resume.0( +; CHECK-LABEL: define dso_local void @test.resume.0( ; CHECK-SAME: {} [[TMP0:%.*]], i32 [[TMP1:%.*]], float [[TMP2:%.*]]) !lgc.cps !0 !continuation !1 { ; CHECK-NEXT: entryresume.0: ; CHECK-NEXT: [[TMP3:%.*]] = call ptr addrspace(32) @lgc.cps.peek(i32 12) -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 2 -; CHECK-NEXT: [[ARG2_RELOAD:%.*]] = load float, ptr addrspace(32) [[TMP4]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 1 -; CHECK-NEXT: [[ARG_RELOAD:%.*]] = load float, ptr addrspace(32) [[TMP5]], align 4 -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 0 -; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[TMP6]], align 4 +; CHECK-NEXT: [[ARG2_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 2 +; CHECK-NEXT: [[ARG2_RELOAD:%.*]] = load float, ptr addrspace(32) [[ARG2_RELOAD_ADDR]], align 4 +; CHECK-NEXT: [[ARG_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 1 +; CHECK-NEXT: [[ARG_RELOAD:%.*]] = load float, ptr addrspace(32) [[ARG_RELOAD_ADDR]], align 4 +; CHECK-NEXT: [[RCR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 0 +; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[RCR_RELOAD_ADDR]], align 4 ; CHECK-NEXT: [[T2:%.*]] = fmul float [[TMP2]], [[ARG_RELOAD]] ; CHECK-NEXT: [[RETURNVALUE:%.*]] = fadd float [[T2]], [[ARG2_RELOAD]] ; CHECK-NEXT: call void @lgc.cps.free(i32 12) diff --git a/shared/continuations/test/lgccps/simple-await.ll b/shared/continuations/test/lgccps/simple-await.ll index ae9dae00fd..c6022cf9a3 100644 --- a/shared/continuations/test/lgccps/simple-await.ll +++ b/shared/continuations/test/lgccps/simple-await.ll @@ -21,26 +21,26 @@ declare void @lgc.cps.jump(...) ; CHECK-SAME: {} [[STATE:%.*]], i32 [[RCR:%.*]], float [[ARG:%.*]]) !lgc.cps !0 !continuation !1 { ; CHECK-NEXT: AllocaSpillBB: ; CHECK-NEXT: [[TMP0:%.*]] = call ptr addrspace(32) @lgc.cps.alloc(i32 8) -; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP0]], i32 0, i32 1 -; CHECK-NEXT: store float [[ARG]], ptr addrspace(32) [[TMP1]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 0 -; CHECK-NEXT: store i32 [[RCR]], ptr addrspace(32) [[TMP2]], align 4 +; CHECK-NEXT: [[ARG_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP0]], i32 0, i32 1 +; CHECK-NEXT: store float [[ARG]], ptr addrspace(32) [[ARG_SPILL_ADDR]], align 4 +; CHECK-NEXT: [[RCR_SPILL_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP0]], i32 0, i32 0 +; CHECK-NEXT: store i32 [[RCR]], ptr addrspace(32) [[RCR_SPILL_ADDR]], align 4 ; CHECK-NEXT: [[T0:%.*]] = fadd float [[ARG]], 1.000000e+00 ; CHECK-NEXT: [[CR:%.*]] = call i32 @lgc.cps.as.continuation.reference(ptr @callee) -; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i32 [[CR]] to ptr -; CHECK-NEXT: [[TMP4:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.0) -; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR]], i32 2, {} poison, i32 [[TMP4]], float [[T0]]) +; CHECK-NEXT: [[TMP1:%.*]] = inttoptr i32 [[CR]] to ptr +; CHECK-NEXT: [[TMP2:%.*]] = call i32 (...) @lgc.cps.as.continuation.reference(ptr @test.resume.0) +; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[CR]], i32 2, {} poison, i32 [[TMP2]], float [[T0]]) ; CHECK-NEXT: unreachable ; ; -; CHECK-LABEL: define void @test.resume.0( +; CHECK-LABEL: define dso_local void @test.resume.0( ; CHECK-SAME: {} [[TMP0:%.*]], i32 [[TMP1:%.*]], float [[TMP2:%.*]]) !lgc.cps !0 !continuation !1 { ; CHECK-NEXT: entryresume.0: ; CHECK-NEXT: [[TMP3:%.*]] = call ptr addrspace(32) @lgc.cps.peek(i32 8) -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 1 -; CHECK-NEXT: [[ARG_RELOAD:%.*]] = load float, ptr addrspace(32) [[TMP4]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 0 -; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[TMP5]], align 4 +; CHECK-NEXT: [[ARG_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME:%.*]], ptr addrspace(32) [[TMP3]], i32 0, i32 1 +; CHECK-NEXT: [[ARG_RELOAD:%.*]] = load float, ptr addrspace(32) [[ARG_RELOAD_ADDR]], align 4 +; CHECK-NEXT: [[RCR_RELOAD_ADDR:%.*]] = getelementptr inbounds [[TEST_FRAME]], ptr addrspace(32) [[TMP3]], i32 0, i32 0 +; CHECK-NEXT: [[RCR_RELOAD:%.*]] = load i32, ptr addrspace(32) [[RCR_RELOAD_ADDR]], align 4 ; CHECK-NEXT: [[RETURNVALUE:%.*]] = fmul float [[TMP2]], [[ARG_RELOAD]] ; CHECK-NEXT: call void @lgc.cps.free(i32 8) ; CHECK-NEXT: call void (...) @lgc.cps.jump(i32 [[RCR_RELOAD]], i32 2, {} poison, i32 poison, float [[RETURNVALUE]]) diff --git a/tool/UpdateTestChecks/common.py b/tool/UpdateTestChecks/common.py index 1f9feb439b..4c8aa2b1f4 100644 --- a/tool/UpdateTestChecks/common.py +++ b/tool/UpdateTestChecks/common.py @@ -639,7 +639,7 @@ def process_run_line(self, function_re, scrubber, raw_tool_output, prefixes, is_ else: # An earlier RUN line used this check prefixes but didn't produce # a body for this function. This happens in Clang tests that use - # preprocesser directives to exclude individual functions from some + # preprocessor directives to exclude individual functions from some # RUN lines. self._func_dict[prefix][func] = None diff --git a/tool/dumper/vkgcPipelineDumper.cpp b/tool/dumper/vkgcPipelineDumper.cpp index 3691e649f7..bb928c58c5 100644 --- a/tool/dumper/vkgcPipelineDumper.cpp +++ b/tool/dumper/vkgcPipelineDumper.cpp @@ -1586,7 +1586,6 @@ void PipelineDumper::updateHashForNonFragmentState(const GraphicsPipelineBuildIn void PipelineDumper::updateHashForFragmentState(const GraphicsPipelineBuildInfo *pipeline, MetroHash64 *hasher) { auto rsState = &pipeline->rsState; hasher->Update(rsState->perSampleShading); - hasher->Update(rsState->provokingVertexMode); hasher->Update(rsState->pixelShaderSamples); // Topology is required when BaryCoord is used diff --git a/util/extensions.txt b/util/extensions.txt index d3cf249784..d65c1fef5c 100644 --- a/util/extensions.txt +++ b/util/extensions.txt @@ -46,3 +46,4 @@ SPV_NV_cooperative_matrix SPV_KHR_cooperative_matrix #endif SPV_NV_shader_atomic_float +SPV_NV_compute_shader_derivatives diff --git a/util/vkgcCapability.h b/util/vkgcCapability.h index 9d2445353f..01604ce415 100644 --- a/util/vkgcCapability.h +++ b/util/vkgcCapability.h @@ -160,6 +160,8 @@ static const char *const VkgcSupportedCapabilities[] = { "CapabilityRayTracingPositionFetchKHR", "CapabilityRayQueryPositionFetchKHR", "CapabilityCooperativeMatrixKHR", + "CapabilityComputeDerivativeGroupLinearNV", + "CapabilityComputeDerivativeGroupQuadsNV", }; }; // namespace Vkgc diff --git a/util/vkgcExtension.cpp b/util/vkgcExtension.cpp index 4e9febd9f7..ce4790b8df 100644 --- a/util/vkgcExtension.cpp +++ b/util/vkgcExtension.cpp @@ -98,6 +98,7 @@ const ExtensionNamePair ExtensionNameTable[ExtensionCount] = { DeclExtensionName(KHR_RAY_TRACING), DeclExtensionName(KHR_RAY_QUERY), DeclExtensionName(NV_SHADER_ATOMIC_FLOAT), + DeclExtensionName(NV_COMPUTE_SHADER_DERIVATIVES), }; // ===================================================================================================================== diff --git a/util/vkgcExtension.h b/util/vkgcExtension.h index dcc7c6679a..a79b06398d 100644 --- a/util/vkgcExtension.h +++ b/util/vkgcExtension.h @@ -80,6 +80,7 @@ enum Extension : unsigned { KHR_RAY_TRACING, KHR_RAY_QUERY, NV_SHADER_ATOMIC_FLOAT, + NV_COMPUTE_SHADER_DERIVATIVES, ExtensionCount, }; diff --git a/util/vkgcUtil.cpp b/util/vkgcUtil.cpp index 0ebe692663..a73e58589f 100644 --- a/util/vkgcUtil.cpp +++ b/util/vkgcUtil.cpp @@ -80,6 +80,21 @@ const char *getShaderStageAbbreviation(ShaderStage shaderStage, bool upper) { return abbr; } +// ===================================================================================================================== +// Wrapper around thread-safe versions of strtok +static char *safeStrtok(char *s, const char *delim, char **savePtr) { + // strtok* and friends are a mess: + // * strtok is not thread-safe in standard C + // * POSIX defined thread-safe strtok_r long ago + // * Microsoft added strtok_s with the same signature and semantics but different name + // * C11 added strtok_s with a different signature from Microsoft's strtok_s +#if defined(_WIN32) + return strtok_s(s, delim, savePtr); +#else + return strtok_r(s, delim, savePtr); +#endif +} + // ===================================================================================================================== // Create directory recursively. // @@ -87,7 +102,8 @@ const char *getShaderStageAbbreviation(ShaderStage shaderStage, bool upper) { bool createDirectory(const char *dir) { namespace fs = std::filesystem; char *dirdup = strdup(dir); - char *token = strtok(dirdup, "/"); + char *saveptr = nullptr; + char *token = safeStrtok(dirdup, "/", &saveptr); bool result = false; @@ -109,7 +125,7 @@ bool createDirectory(const char *dir) { } } - token = strtok(NULL, "/"); + token = safeStrtok(NULL, "/", &saveptr); tmp += "/"; } diff --git a/version/CMakeLists.txt b/version/CMakeLists.txt new file mode 100644 index 0000000000..6d73ed02a5 --- /dev/null +++ b/version/CMakeLists.txt @@ -0,0 +1,101 @@ +## + ####################################################################################################################### + # + # Copyright (c) 2017-2023 Advanced Micro Devices, Inc. All Rights Reserved. + # + # Permission is hereby granted, free of charge, to any person obtaining a copy + # of this software and associated documentation files (the "Software"), to deal + # in the Software without restriction, including without limitation the rights + # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + # copies of the Software, and to permit persons to whom the Software is + # furnished to do so, subject to the following conditions: + # + # The above copyright notice and this permission notice shall be included in all + # copies or substantial portions of the Software. + # + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + # SOFTWARE. + # + ####################################################################################################################### + +### llpc_version library ############################################################################################### +add_library(llpc_version INTERFACE) + +target_include_directories(llpc_version INTERFACE + $) + +configure_file(include/llpcVersion.h.in include/llpcVersion.h) + +if (NOT LLPC_CLIENT_INTERFACE_MAJOR_VERSION) + message(FATAL_ERROR "Client of LLPC must set LLPC_CLIENT_INTERFACE_MAJOR_VERSION") +endif() + +target_compile_definitions(llpc_version INTERFACE + LLPC_CLIENT_INTERFACE_MAJOR_VERSION=${LLPC_CLIENT_INTERFACE_MAJOR_VERSION} + # Hardcode the endian-ness define. Our register headers expect it anyway + LITTLEENDIAN_CPU + CHIP_HDR_NAVI14 + CHIP_HDR_NAVI21 + CHIP_HDR_NAVI22 + CHIP_HDR_NAVI23 + CHIP_HDR_NAVI24 +#if LLPC_BUILD_NAVI31 + CHIP_HDR_NAVI31 +#endif +#if LLPC_BUILD_NAVI33 + CHIP_HDR_NAVI33 +#endif + CHIP_HDR_RENOIR) + +# Mark llpc_version as an installable target for which *.cmake files are created +# that a hypothetical external user could include to import the target, either +# from an installed version or from the build-tree. +# +# This is not expected to actually work (least of all because the generated +# llpcVersion.h is not installed), but this target is linked to by targets that +# we register as LLVM projects (LGC, Continuations, ...) and the LLVM CMake +# infrastructure believes that those should be installable and exported, which +# means CMake also requires llpc_version to be installable and exported. +# +# This would not be required if everybody linked against llpc_version PRIVATEly, +# but then that would create a footgun if any of the version definitions are +# used in header files. Better to have this weird hack and have everybody link +# against llpc_version PUBLICly. +install(TARGETS llpc_version EXPORT llpc_version-targets) +install(EXPORT llpc_version-targets DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake) # installed version +export(TARGETS llpc_version FILE llpc_version.cmake) # build-tree version + +### Cached Config-related Options ###################################################################################### +option(LLPC_BUILD_NAVI12 "LLPC support for NAVI12?" ON) +if (LLPC_BUILD_NAVI12) + target_compile_definitions(llpc_version INTERFACE LLPC_BUILD_NAVI12 CHIP_HDR_NAVI12) +endif() +#if LLPC_BUILD_NAVI32 +option(LLPC_BUILD_NAVI32 "LLPC support for NAVI32?" ON) +if (LLPC_BUILD_NAVI32) + target_compile_definitions(llpc_version INTERFACE LLPC_BUILD_NAVI32 CHIP_HDR_NAVI32) +endif() +#endif +option(LLPC_BUILD_REMBRANDT "LLPC support for REMBRANDT?" ON) +if (LLPC_BUILD_REMBRANDT) + target_compile_definitions(llpc_version INTERFACE LLPC_BUILD_REMBRANDT CHIP_HDR_REMBRANDT) +endif() +option(LLPC_BUILD_RAPHAEL "LLPC support for RAPHAEL?" ON) +if (LLPC_BUILD_RAPHAEL) + target_compile_definitions(llpc_version INTERFACE LLPC_BUILD_RAPHAEL CHIP_HDR_RAPHAEL) +endif() +option(LLPC_BUILD_MENDOCINO "LLPC support for MENDOCINO?" ON) +if (LLPC_BUILD_MENDOCINO) + target_compile_definitions(llpc_version INTERFACE LLPC_BUILD_MENDOCINO CHIP_HDR_MENDOCINO) +endif() +#if LLPC_BUILD_PHOENIX1 +option(LLPC_BUILD_PHOENIX1 "LLPC support for PHOENIX1?" ON) +if (LLPC_BUILD_PHOENIX1) + target_compile_definitions(llpc_version INTERFACE LLPC_BUILD_PHOENIX1 CHIP_HDR_PHOENIX1) +endif() +#endif diff --git a/version/include/llpcVersion.h.in b/version/include/llpcVersion.h.in new file mode 100644 index 0000000000..80a289b916 --- /dev/null +++ b/version/include/llpcVersion.h.in @@ -0,0 +1,184 @@ +/* + *********************************************************************************************************************** + * + * Copyright (c) 2020-2023 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + **********************************************************************************************************************/ +/** + *********************************************************************************************************************** + * @file llpcVersion.h + * @brief LLPC_*_INTERFACE_*_VERSION defines, checks, and related documentation + * + * Auto-generated during CMake configuration based on the LLPC_CLIENT_INTERFACE_MAJOR_VERSION variable set by the + * driver CMake files. + *********************************************************************************************************************** + */ + +// ------------------------------------------------------------------------------------------------------------------- +// @page VersionHistory +// %Version History +// | %Version | Change Description | +// | -------- | ----------------------------------------------------------------------------------------------------- | +// | 70.2 | Add useSoftwareVertexBufferDescriptors to GraphicsPipelineBuildInfo | +// | 70.1 | Add cpsFlags to RayTracingPipelineBuildInfo | +// | 70.0 | Add enablePrimGeneratedQuery to PipelineOptions | +// | 69.1 | Add useBarycentric to ShaderModuleUsage | +// | 69.0 | Enable continuations transform in LLPC | +// | 68.0 | Remove ICache *cache in all PipelineBuildInfo | +// | 67.0 | Modify the uber fetch shader. Adds locationMask(64bit) at the beginning of uber fetch shader internal | +// | | buffer which flags whether the related attribute data is valid. | +// | 66.1 | Add forwardPropagateNoContract and backwardPropagateNoContract to PipelineShaderOptions | +// | 66.0 | Remove shader cache in LLPC | +// | 65.6 | Add Result::RequireFullPipeline, returned if unlink shader fails. | +// | 65.5 | Rename noContract in PipelineShaderOptions to noContractOpDot | +// | 65.4 | Add disableSampleMask to PipelineOptions | +// | 65.3 | Add originUpperLeft to GraphicsPipelineBuildInfo | +// | 65.2 | Support SPIRV extended vertex attribute formats during vertex fetch module. | +// | 65.0 | Remove updateDescInElf | +// | 64.2 | Add dynamicSampleInfo to GraphicsPipelineBuildInfo::rsState | +// | 64.1 | Add disableTruncCoordForGather to PipelineOptions. | +// | 64.0 | Add enableColorExportShader to GraphicsPipelineBuildInfo. | +// | 63.3 | Add TessellationLevel to iaState | +// | 63.2 | Add vertex64BitsAttribSingleLoc to PipelineOptions | +// | 63.1 | Add forceDisableStreamOut and forceEnablePrimStats to ApiXfbOutData | +// | 63.0 | Add Atomic Counter, its default descriptor and map its concreteType to Buffer. | +// | 62.1 | Add ApiXfbOutData GraphicsPipelineBuildInfo | +// | 62.0 | Default to the compiler getting the GPURT library directly, and move shader library info into RtState | +// | 61.16| Add replaceSetWithResourceType to PipelineOptions | +// | 61.15| Add disableReadFirstLaneWorkaround to PipelineShaderOptions | +// | 61.14| Add rasterStream to rsState +// | 61.13| Add dualSourceBlendDynamic to cbState | +// | 61.12| Add mode to RayTracingPipelineBuildInfo | +// | 61.11| Add UniformConstantMap and related structures | +// | 61.10| Add useShadingRate and useSampleInfo to ShaderModuleUsage | +// | 61.8 | Add enableImplicitInvariantExports to PipelineOptions | +// | 61.7 | Add disableFMA to PipelineShaderOptions | +// | 61.6 | Add workaroundInitializeOutputsToZero to PipelineShaderOptions | +// | 61.5 | Add RtIpVersion (including its checkers) to represent RT IP | +// | 61.4 | Add workaroundStorageImageFormats to PipelineShaderOptions | +// | 61.2 | Add pClientMetadata and clientMetadataSize to all PipelineBuildInfos | +// | 61.1 | Add IPipelineDumper::GetGraphicsShaderBinaryHash | +// | 61.0 | Add DescriptorMutable type and ResourceMappingNode::strideInDwords to support mutable descriptors | +// | 60.0 | Simplify the enum NggCompactMode to a boolean flag | +// | 59.0 | Remove the option enableVertexReuse from NggState | +// | 57.2 | Move all internal resource binding id to enum InternalBinding. | +// | 57.1 | Add forceNonUniformResourceIndexStageMask to PipelineOptions | +// | 57.0 | Merge aggressiveInvariantLoads and disableInvariantLoads to an enumerated option | +// | 56.2 | Add aggressiveInvariantLoads and disableInvariantLoads to PipelineShaderOptions | +// | 56.1 | Add struct UberFetchShaderAttribInfo | +// | 56.0 | Move maxRayLength to RtState | +// | 55.2 | Add pipeline layout API hash to all PipelineBuildInfos | +// | 55.1 | Add nsaThreshold to PipelineShaderOptions | +// | 55.0 | Remove isInternalRtShader from module options | +// | 54.9 | Add internalRtShaders to PipelineOptions to allow for dumping this data | +// | 54.6 | Add reverseThreadGroup to PipelineOptions | +// | 54.5 | Add forceLateZ to PipelineShaderOptions | +// | 54.4 | Add isReplay to RayTracingPipelineBuildInfo for ray tracing capture replay feature | +// | 54.3 | Add usePointSize to ShaderModuleUsage | +// | 54.2 | Add subgroupSize to PipelineShaderOptions | +// | 54.1 | Add overrideForceThreadIdSwizzling overrideShaderThreadGroupSizeX, overrideShaderThreadGroupSizeY | +// | | and overrideShaderThreadGroupSizeZ to PipelineShaderOptions | +// | 54.0 | Add overrideThreadGroupSizeX, overrideThreadGroupSizeY and overrideThreadGroupSizeZ to PipelineOptions| +// | 53.7 | Add threadGroupSwizzleMode to PipelineOptions | +// | 53.6 | Add scalarizeWaterfallLoads to PipelineShaderOptions | +// | 53.5 | Add forceCsThreadIdSwizzling for thread id swizzle in 8*4 | +// | 53.4 | Add ldsSpillLimitDwords shader option | +// | 53.3 | Add disableFastMathFlags shader option, plus support for this and fastMathFlags in pipeline files | +// | 53.2 | Add resourceLayoutScheme to PipelineOptions | +// | 53.1 | Add PartPipelineStage enum for part-pipeline mode | +// | 53.0 | Add optimizationLevel to PipelineOptions | +// | 52.3 | Add fastMathFlags to PipelineShaderOptions | +// | 52.2 | Add provokingVertexMode to rsState | +// | 52.1 | Add pageMigrationEnabled to PipelineOptions | +// | 52.0 | Add the member word4 and word5 to SamplerYCbCrConversionMetaData | +// | 51.2 | Added new pipeline shader info to support mesh shader | +// | 51.0 | Added new shader stage enumerates to support mesh shader | +// | 50.2 | Add the member dsState to GraphicsPipelineBuildInfo | +// | 50.1 | Disclose ResourceMappingNodeType::InlineBuffer | +// | 50.0 | Removed the member 'enableOpt' of ShaderModuleOptions | +// | 49.1 | Added enableEarlyCompile to GraphicsPipelineBuildInfo | +// | 49.0 | Added DescriptorConstBuffer, DescriptorConstBufferCompact, DescriptorImage, DescriptorConstTexelBuffer| +// | | to ResourceMappingNodeType | +// | 48.1 | Added enableUberFetchShader to GraphicsPipelineBuildInfo | +// | 48.0 | Removed the member 'polygonMode' of rsState | +// | 47.0 | Always get culling controls from primitive shader table | +// | 46.3 | Added enableInterpModePatch to PipelineOptions | +// | 46.2 | Added optimizeTessFactor to PipelineOptions | +// | 46.1 | Added dynamicVertexStride to GraphicsPipelineBuildInfo | +// | 46.0 | Removed the member 'depthBiasEnable' of rsState | +// | 45.5 | Added new enum type ThreadGroupSwizzleMode for thread group swizzling for compute shaders | +// | 45.4 | Added disableLicmThreshold, unrollHintThreshold, and dontUnrollHintThreshold to PipelineShaderOptions | +// | 45.3 | Add pipelinedump function to enable BeginPipelineDump and GetPipelineName | +// | 45.2 | Add GFX IP plus checker to GfxIpVersion | +// | 45.1 | Add pipelineCacheAccess, stageCacheAccess(es) to GraphicsPipelineBuildOut/ComputePipelineBuildOut | +// | 45.0 | Remove the member 'enableFastLaunch' of NGG state | +// | 44.0 | Rename the member 'forceNonPassthrough' of NGG state to 'forceCullingMode' | +// | 43.1 | Add disableImageResourceCheck in PipelineOptions | +// | 43.0 | Removed the enumerate WaveBreakSize::DrawTime | +// | 42.0 | Removed tileOptimal flag from SamplerYcbcrConversion metadata struct | +// | 41.0 | Moved resource mapping from ShaderPipeline-level to Pipeline-level | +// | 40.4 | Added fp32DenormalMode in PipelineShaderOptions to allow overriding SPIR-V denormal settings | +// | 40.3 | Added ICache interface | +// | 40.2 | Added extendedRobustness in PipelineOptions to support VK_EXT_robustness2 | +// | 40.1 | Added disableLoopUnroll to PipelineShaderOptions | +// | 40.0 | Added DescriptorReserved12, which moves DescriptorYCbCrSampler down to 13 | +// | 39.0 | Non-LLPC-specific XGL code should #include vkgcDefs.h instead of llpc.h | +// | 38.3 | Added shadowDescriptorTableUsage and shadowDescriptorTablePtrHigh to PipelineOptions | +// | 38.2 | Added scalarThreshold to PipelineShaderOptions | +// | 38.1 | Added unrollThreshold to PipelineShaderOptions | +// | 38.0 | Removed CreateShaderCache in ICompiler and pShaderCache in pipeline build info | +// | 37.0 | Removed the -enable-dynamic-loop-unroll option | +// | 36.0 | Add 128 bit hash as clientHash in PipelineShaderOptions | +// | 35.0 | Added disableLicm to PipelineShaderOptions | +// | 33.0 | Add enableLoadScalarizer option into PipelineShaderOptions. | +// | 32.0 | Add ShaderModuleOptions in ShaderModuleBuildInfo | +// | 31.0 | Add PipelineShaderOptions::allowVaryWaveSize | +// | 30.0 | Removed PipelineOptions::autoLayoutDesc | +// | 28.0 | Added reconfigWorkgroupLayout to PipelineOptions and useSiScheduler to PipelineShaderOptions | +// | 27.0 | Remove the includeIrBinary option from PipelineOptions as only IR disassembly is now dumped | +// | 25.0 | Add includeIrBinary option into PipelineOptions for including IR binaries into ELF files. | +// | 24.0 | Add forceLoopUnrollCount option into PipelineShaderOptions. | +// | 23.0 | Add flag robustBufferAccess in PipelineOptions to check out of bounds of private array. | +// | 22.0 | Internal revision. | +// | 21.0 | Add stage in Pipeline shader info and struct PipelineBuildInfo to simplify pipeline dump interface. | +// | -------- | ----------------------------------------------------------------------------------------------------- | +// + +#pragma once + +/// LLPC major interface version. +#define LLPC_INTERFACE_MAJOR_VERSION 70 + +/// LLPC minor interface version. +#define LLPC_INTERFACE_MINOR_VERSION 2 + +/// The client's LLPC major interface version +#ifndef LLPC_CLIENT_INTERFACE_MAJOR_VERSION +#cmakedefine LLPC_CLIENT_INTERFACE_MAJOR_VERSION @LLPC_CLIENT_INTERFACE_MAJOR_VERSION@ +#endif + +#ifndef LLPC_CLIENT_INTERFACE_MAJOR_VERSION +#error Client of LLPC must set LLPC_CLIENT_INTERFACE_MAJOR_VERSION +#endif + +#if LLPC_CLIENT_INTERFACE_MAJOR_VERSION < 60 +#error LLPC client version is too old +#endif