Update llpc from commit a1369c61

GPUOpen-Drivers · Feb 22, 2024 · 5579b5e · 5579b5e
1 parent 5de2a34
commit 5579b5e
Show file tree

Hide file tree

Showing 210 changed files with 2,453 additions and 3,367 deletions.
diff --git a/compilerutils/lib/CompilerUtils.cpp b/compilerutils/lib/CompilerUtils.cpp
@@ -299,14 +299,32 @@ CompilerUtils::CrossModuleInliner::inlineCall(IRBuilder<> &b, llvm::Function *ca
   if (!callee->getReturnType()->isVoidTy())
     fakeUse = cast<FreezeInst>(b.CreateFreeze(call));
 
+  // If the builder is at the end of the basic block then we don't have complete IR yet. We need some placeholder to
+  // know where to reset the insert point to.
+  Instruction *insertPointMarker = nullptr;
+  if (b.GetInsertPoint() == b.GetInsertBlock()->end()) {
+    assert(!b.GetInsertBlock()->getTerminator());
+    if (fakeUse)
+      insertPointMarker = fakeUse;
+    else
+      insertPointMarker = b.CreateUnreachable();
+  }
+
   auto newBBs = inlineCall(*call);
 
+  if (insertPointMarker) {
+    b.SetInsertPoint(insertPointMarker->getParent());
+    if (insertPointMarker != fakeUse)
+      insertPointMarker->eraseFromParent();
+  } else {
+    b.SetInsertPoint(&*b.GetInsertPoint());
+  }
+
   Value *result = nullptr;
   if (fakeUse) {
     result = fakeUse->getOperand(0);
     fakeUse->eraseFromParent();
   }
-  b.SetInsertPoint(&*b.GetInsertPoint());
   return {result, newBBs};
 }
 

diff --git a/include/khronos/spirv/spirv.hpp b/include/khronos/spirv/spirv.hpp
@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2020 The Khronos Group Inc.
+// Copyright (c) 2014-2024 The Khronos Group Inc.
 // 
 // Permission is hereby granted, free of charge, to any person obtaining a copy
 // of this software and/or associated documentation files (the "Materials"),
@@ -73,6 +73,7 @@ enum SourceLanguage {
     SourceLanguageNZSL = 9,
     SourceLanguageWGSL = 10,
     SourceLanguageSlang = 11,
+    SourceLanguageZig = 12,
     SourceLanguageMax = 0x7fffffff,
 };
 
@@ -183,6 +184,8 @@ enum ExecutionMode {
     ExecutionModeStencilRefUnchangedBackAMD = 5082,
     ExecutionModeStencilRefGreaterBackAMD = 5083,
     ExecutionModeStencilRefLessBackAMD = 5084,
+    ExecutionModeQuadDerivativesKHR = 5088,
+    ExecutionModeRequireFullQuadsKHR = 5089,
     ExecutionModeOutputLinesEXT = 5269,
     ExecutionModeOutputLinesNV = 5269,
     ExecutionModeOutputPrimitivesEXT = 5270,
@@ -207,6 +210,8 @@ enum ExecutionMode {
     ExecutionModeNoGlobalOffsetINTEL = 5895,
     ExecutionModeNumSIMDWorkitemsINTEL = 5896,
     ExecutionModeSchedulerTargetFmaxMhzINTEL = 5903,
+    ExecutionModeMaximallyReconvergesKHR = 6023,
+    ExecutionModeFPFastMathDefault = 6028,
     ExecutionModeStreamingInterfaceINTEL = 6154,
     ExecutionModeRegisterMapInterfaceINTEL = 6160,
     ExecutionModeNamedBarrierCountINTEL = 6417,
@@ -426,8 +431,11 @@ enum FPFastMathModeShift {
     FPFastMathModeNSZShift = 2,
     FPFastMathModeAllowRecipShift = 3,
     FPFastMathModeFastShift = 4,
+    FPFastMathModeAllowContractShift = 16,
     FPFastMathModeAllowContractFastINTELShift = 16,
+    FPFastMathModeAllowReassocShift = 17,
     FPFastMathModeAllowReassocINTELShift = 17,
+    FPFastMathModeAllowTransformShift = 18,
     FPFastMathModeMax = 0x7fffffff,
 };
 
@@ -438,8 +446,11 @@ enum FPFastMathModeMask {
     FPFastMathModeNSZMask = 0x00000004,
     FPFastMathModeAllowRecipMask = 0x00000008,
     FPFastMathModeFastMask = 0x00000010,
+    FPFastMathModeAllowContractMask = 0x00010000,
     FPFastMathModeAllowContractFastINTELMask = 0x00010000,
+    FPFastMathModeAllowReassocMask = 0x00020000,
     FPFastMathModeAllowReassocINTELMask = 0x00020000,
+    FPFastMathModeAllowTransformMask = 0x00040000,
 };
 
 enum FPRoundingMode {
@@ -583,6 +594,9 @@ enum Decoration {
     DecorationMergeINTEL = 5834,
     DecorationBankBitsINTEL = 5835,
     DecorationForcePow2DepthINTEL = 5836,
+    DecorationStridesizeINTEL = 5883,
+    DecorationWordsizeINTEL = 5884,
+    DecorationTrueDualPortINTEL = 5885,
     DecorationBurstCoalesceINTEL = 5899,
     DecorationCacheSizeINTEL = 5900,
     DecorationDontStaticallyCoalesceINTEL = 5901,
@@ -601,9 +615,7 @@ enum Decoration {
     DecorationSingleElementVectorINTEL = 6085,
     DecorationVectorComputeCallableFunctionINTEL = 6087,
     DecorationMediaBlockIOINTEL = 6140,
-    DecorationInitModeINTEL = 6147,
-    DecorationImplementInRegisterMapINTEL = 6148,
-    DecorationHostAccessINTEL = 6168,
+    DecorationStallFreeINTEL = 6151,
     DecorationFPMaxErrorDecorationINTEL = 6170,
     DecorationLatencyControlLabelINTEL = 6172,
     DecorationLatencyControlConstraintINTEL = 6173,
@@ -616,6 +628,9 @@ enum Decoration {
     DecorationMMHostInterfaceMaxBurstINTEL = 6181,
     DecorationMMHostInterfaceWaitRequestINTEL = 6182,
     DecorationStableKernelArgumentINTEL = 6183,
+    DecorationHostAccessINTEL = 6188,
+    DecorationInitModeINTEL = 6190,
+    DecorationImplementInRegisterMapINTEL = 6191,
     DecorationCacheControlLoadINTEL = 6442,
     DecorationCacheControlStoreINTEL = 6443,
     DecorationMax = 0x7fffffff,
@@ -1068,6 +1083,7 @@ enum Capability {
     CapabilityInt64ImageEXT = 5016,
     CapabilityShaderClockKHR = 5055,
     CapabilityShaderEnqueueAMDX = 5067,
+    CapabilityQuadControlKHR = 5087,
     CapabilitySampleMaskOverrideCoverageNV = 5249,
     CapabilityGeometryShaderPassthroughNV = 5251,
     CapabilityShaderViewportIndexLayerEXT = 5254,
@@ -1187,21 +1203,24 @@ enum Capability {
     CapabilityCooperativeMatrixKHR = 6022,
     CapabilityBitInstructions = 6025,
     CapabilityGroupNonUniformRotateKHR = 6026,
+    CapabilityFloatControls2 = 6029,
     CapabilityAtomicFloat32AddEXT = 6033,
     CapabilityAtomicFloat64AddEXT = 6034,
-    CapabilityLongConstantCompositeINTEL = 6089,
+    CapabilityLongCompositesINTEL = 6089,
     CapabilityOptNoneINTEL = 6094,
     CapabilityAtomicFloat16AddEXT = 6095,
     CapabilityDebugInfoModuleINTEL = 6114,
     CapabilityBFloat16ConversionINTEL = 6115,
     CapabilitySplitBarrierINTEL = 6141,
-    CapabilityGlobalVariableFPGADecorationsINTEL = 6146,
+    CapabilityFPGAClusterAttributesV2INTEL = 6150,
     CapabilityFPGAKernelAttributesv2INTEL = 6161,
-    CapabilityGlobalVariableHostAccessINTEL = 6167,
     CapabilityFPMaxErrorINTEL = 6169,
     CapabilityFPGALatencyControlINTEL = 6171,
     CapabilityFPGAArgumentInterfacesINTEL = 6174,
+    CapabilityGlobalVariableHostAccessINTEL = 6187,
+    CapabilityGlobalVariableFPGADecorationsINTEL = 6189,
     CapabilityGroupUniformArithmeticKHR = 6400,
+    CapabilityMaskedGatherScatterINTEL = 6427,
     CapabilityCacheControlsINTEL = 6441,
     CapabilityMax = 0x7fffffff,
 };
@@ -1774,6 +1793,8 @@ enum Op {
     OpFinalizeNodePayloadsAMDX = 5075,
     OpFinishWritingNodePayloadAMDX = 5078,
     OpInitializeNodePayloadsAMDX = 5090,
+    OpGroupNonUniformQuadAllKHR = 5110,
+    OpGroupNonUniformQuadAnyKHR = 5111,
     OpHitObjectRecordHitMotionNV = 5249,
     OpHitObjectRecordHitWithIndexMotionNV = 5250,
     OpHitObjectRecordMissMotionNV = 5251,
@@ -2083,6 +2104,7 @@ enum Op {
     OpTypeStructContinuedINTEL = 6090,
     OpConstantCompositeContinuedINTEL = 6091,
     OpSpecConstantCompositeContinuedINTEL = 6092,
+    OpCompositeConstructContinuedINTEL = 6096,
     OpConvertFToBF16INTEL = 6116,
     OpConvertBF16ToFINTEL = 6117,
     OpControlBarrierArriveINTEL = 6142,
@@ -2095,6 +2117,8 @@ enum Op {
     OpGroupLogicalAndKHR = 6406,
     OpGroupLogicalOrKHR = 6407,
     OpGroupLogicalXorKHR = 6408,
+    OpMaskedGatherINTEL = 6428,
+    OpMaskedScatterINTEL = 6429,
     OpMax = 0x7fffffff,
 };
 
@@ -2502,6 +2526,8 @@ inline void HasResultAndType(Op opcode, bool *hasResult, bool *hasResultType) {
     case OpFinalizeNodePayloadsAMDX: *hasResult = false; *hasResultType = false; break;
     case OpFinishWritingNodePayloadAMDX: *hasResult = true; *hasResultType = true; break;
     case OpInitializeNodePayloadsAMDX: *hasResult = false; *hasResultType = false; break;
+    case OpGroupNonUniformQuadAllKHR: *hasResult = true; *hasResultType = true; break;
+    case OpGroupNonUniformQuadAnyKHR: *hasResult = true; *hasResultType = true; break;
     case OpHitObjectRecordHitMotionNV: *hasResult = false; *hasResultType = false; break;
     case OpHitObjectRecordHitWithIndexMotionNV: *hasResult = false; *hasResultType = false; break;
     case OpHitObjectRecordMissMotionNV: *hasResult = false; *hasResultType = false; break;
@@ -2806,6 +2832,7 @@ inline void HasResultAndType(Op opcode, bool *hasResult, bool *hasResultType) {
     case OpTypeStructContinuedINTEL: *hasResult = false; *hasResultType = false; break;
     case OpConstantCompositeContinuedINTEL: *hasResult = false; *hasResultType = false; break;
     case OpSpecConstantCompositeContinuedINTEL: *hasResult = false; *hasResultType = false; break;
+    case OpCompositeConstructContinuedINTEL: *hasResult = true; *hasResultType = true; break;
     case OpConvertFToBF16INTEL: *hasResult = true; *hasResultType = true; break;
     case OpConvertBF16ToFINTEL: *hasResult = true; *hasResultType = true; break;
     case OpControlBarrierArriveINTEL: *hasResult = false; *hasResultType = false; break;
@@ -2818,6 +2845,8 @@ inline void HasResultAndType(Op opcode, bool *hasResult, bool *hasResultType) {
     case OpGroupLogicalAndKHR: *hasResult = true; *hasResultType = true; break;
     case OpGroupLogicalOrKHR: *hasResult = true; *hasResultType = true; break;
     case OpGroupLogicalXorKHR: *hasResult = true; *hasResultType = true; break;
+    case OpMaskedGatherINTEL: *hasResult = true; *hasResultType = true; break;
+    case OpMaskedScatterINTEL: *hasResult = false; *hasResultType = false; break;
     }
 }
 #endif /* SPV_ENABLE_UTILITY_CODE */

diff --git a/include/vkgcDefs.h b/include/vkgcDefs.h
@@ -1175,6 +1175,8 @@ struct GraphicsPipelineBuildInfo {
 #if LLPC_CLIENT_INTERFACE_MAJOR_VERSION < 68
   ICache *cache; ///< ICache, used to search for the compiled shader data
 #endif
+  uint64_t pipelineApiHash; ///< Pipeline hash. If non-zero, this will be used directly as stable hash
+
   PipelineShaderInfo task; ///< Task shader
   PipelineShaderInfo vs;   ///< Vertex shader
   PipelineShaderInfo tcs;  ///< Tessellation control shader
@@ -1267,6 +1269,7 @@ struct ComputePipelineBuildInfo {
 #if LLPC_CLIENT_INTERFACE_MAJOR_VERSION < 68
   ICache *cache; ///< ICache, used to search for the compiled shader data
 #endif
+  uint64_t pipelineApiHash;            ///< Pipeline hash. If non-zero, this will be used directly as stable hash
   unsigned deviceIndex;                ///< Device index for device group
   PipelineShaderInfo cs;               ///< Compute shader
   ResourceMappingData resourceMapping; ///< Resource mapping graph and static descriptor values
@@ -1290,13 +1293,14 @@ struct RayTracingPipelineBuildInfo {
 #if LLPC_CLIENT_INTERFACE_MAJOR_VERSION < 68
   ICache *cache; ///< ICache, used to search for the compiled shader data
 #endif
-  unsigned deviceIndex;                                      ///< Device index for device group
-  unsigned deviceCount;                                      ///< Device count for device group
-  unsigned shaderCount;                                      ///< Count of shader info
-  PipelineShaderInfo *pShaders;                              ///< An array of shader info
-  ResourceMappingData resourceMapping;                       ///< Resource mapping graph and static descriptor values
-  uint64_t pipelineLayoutApiHash;                            ///< Pipeline Layout Api Hash
-  unsigned shaderGroupCount;                                 ///< Count of shader group
+  uint64_t pipelineApiHash;            ///< Pipeline hash. If non-zero, this will be used directly as stable hash
+  unsigned deviceIndex;                ///< Device index for device group
+  unsigned deviceCount;                ///< Device count for device group
+  unsigned shaderCount;                ///< Count of shader info
+  PipelineShaderInfo *pShaders;        ///< An array of shader info
+  ResourceMappingData resourceMapping; ///< Resource mapping graph and static descriptor values
+  uint64_t pipelineLayoutApiHash;      ///< Pipeline Layout Api Hash
+  unsigned shaderGroupCount;           ///< Count of shader group
   const VkRayTracingShaderGroupCreateInfoKHR *pShaderGroups; ///< An array of shader group
   LibraryMode libraryMode;                                   ///< Whether to compile as pipeline or library or both
   unsigned libraryCount;                                     ///< Count of libraries linked into this build

diff --git a/lgc/CMakeLists.txt b/lgc/CMakeLists.txt
@@ -128,7 +128,6 @@ target_sources(LLVMlgc PRIVATE
 target_sources(LLVMlgc PRIVATE
     elfLinker/ColorExportShader.cpp
     elfLinker/ElfLinker.cpp
-    elfLinker/FetchShader.cpp
     elfLinker/GlueShader.cpp
     elfLinker/NullFragmentShader.cpp
 )
@@ -142,6 +141,7 @@ target_sources(LLVMlgc PRIVATE
     patch/Gfx9ConfigBuilder.cpp
     patch/LowerDebugPrintf.cpp
     patch/LowerDesc.cpp
+    patch/LowerSubgroupOps.cpp
     patch/MeshTaskShader.cpp
     patch/NggPrimShader.cpp
     patch/Patch.cpp

diff --git a/lgc/builder/BuilderRecorder.cpp b/lgc/builder/BuilderRecorder.cpp
@@ -260,12 +260,8 @@ StringRef BuilderRecorder::getCallName(BuilderOpcode opcode) {
     return "get.wave.size";
   case BuilderOpcode::GetSubgroupSize:
     return "get.subgroup.size";
-  case BuilderOpcode::SubgroupElect:
-    return "subgroup.elect";
   case BuilderOpcode::SubgroupAll:
     return "subgroup.all";
-  case BuilderOpcode::SubgroupAny:
-    return "subgroup.any";
   case BuilderOpcode::SubgroupAllEqual:
     return "subgroup.all.equal";
   case BuilderOpcode::SubgroupRotate:
@@ -1620,14 +1616,6 @@ Value *Builder::CreateGetSubgroupSize(const Twine &instName) {
   return record(BuilderOpcode::GetSubgroupSize, getInt32Ty(), {}, instName);
 }
 
-// =====================================================================================================================
-// Create a subgroup elect.
-//
-// @param instName : Name to give instruction(s)
-Value *Builder::CreateSubgroupElect(const Twine &instName) {
-  return record(BuilderOpcode::SubgroupElect, getInt1Ty(), {}, instName);
-}
-
 // =====================================================================================================================
 // Create a subgroup all.
 //
@@ -1637,15 +1625,6 @@ Value *Builder::CreateSubgroupAll(Value *const value, const Twine &instName) {
   return record(BuilderOpcode::SubgroupAll, getInt1Ty(), {value}, instName);
 }
 
-// =====================================================================================================================
-// Create a subgroup any
-//
-// @param value : The value to compare
-// @param instName : Name to give instruction(s)
-Value *Builder::CreateSubgroupAny(Value *const value, const Twine &instName) {
-  return record(BuilderOpcode::SubgroupAny, getInt1Ty(), {value}, instName);
-}
-
 // =====================================================================================================================
 // Create a subgroup all equal.
 //
@@ -2126,7 +2105,6 @@ Instruction *Builder::record(BuilderOpcode opcode, Type *resultTy, ArrayRef<Valu
     case BuilderOpcode::SubgroupAll:
     case BuilderOpcode::SubgroupAllEqual:
     case BuilderOpcode::SubgroupRotate:
-    case BuilderOpcode::SubgroupAny:
     case BuilderOpcode::SubgroupBallot:
     case BuilderOpcode::SubgroupBroadcast:
     case BuilderOpcode::SubgroupBroadcastWaterfall:
@@ -2135,7 +2113,6 @@ Instruction *Builder::record(BuilderOpcode opcode, Type *resultTy, ArrayRef<Valu
     case BuilderOpcode::SubgroupClusteredMultiExclusive:
     case BuilderOpcode::SubgroupClusteredInclusive:
     case BuilderOpcode::SubgroupClusteredReduction:
-    case BuilderOpcode::SubgroupElect:
     case BuilderOpcode::SubgroupInverseBallot:
     case BuilderOpcode::SubgroupMbcnt:
     case BuilderOpcode::SubgroupPartition:

diff --git a/lgc/builder/BuilderRecorder.h b/lgc/builder/BuilderRecorder.h
@@ -160,9 +160,7 @@ enum BuilderOpcode : unsigned {
 
   // Subgroup
   GetSubgroupSize,
-  SubgroupElect,
   SubgroupAll,
-  SubgroupAny,
   SubgroupAllEqual,
   SubgroupRotate,
   SubgroupBroadcast,

diff --git a/lgc/builder/BuilderReplayer.cpp b/lgc/builder/BuilderReplayer.cpp
@@ -50,16 +50,7 @@ using namespace llvm;
 // @returns : The preserved analyses (The analyses that are still valid after this pass)
 PreservedAnalyses BuilderReplayer::run(Module &module, ModuleAnalysisManager &analysisManager) {
   PipelineState *pipelineState = analysisManager.getResult<PipelineStateWrapper>(module).getPipelineState();
-  runImpl(module, pipelineState);
-  return PreservedAnalyses::none();
-}
 
-// =====================================================================================================================
-// Run the BuilderReplayer pass on a module
-//
-// @param module : Module to run this pass on
-// @returns : True if the module was modified by the transformation and false otherwise
-bool BuilderReplayer::runImpl(Module &module, PipelineState *pipelineState) {
   LLVM_DEBUG(dbgs() << "Running the pass of replaying LLPC builder calls\n");
 
   // Set up the pipeline state from the specified linked IR module.
@@ -121,7 +112,7 @@ bool BuilderReplayer::runImpl(Module &module, PipelineState *pipelineState) {
     func->eraseFromParent();
   m_builder = nullptr;
 
-  return true;
+  return PreservedAnalyses::none();
 }
 
 // =====================================================================================================================
@@ -728,15 +719,9 @@ Value *BuilderReplayer::processCall(unsigned opcode, CallInst *call) {
   case BuilderOpcode::GetSubgroupSize: {
     return m_builder->CreateGetSubgroupSize();
   }
-  case BuilderOpcode::SubgroupElect: {
-    return m_builder->CreateSubgroupElect();
-  }
   case BuilderOpcode::SubgroupAll: {
     return m_builder->CreateSubgroupAll(args[0]);
   }
-  case BuilderOpcode::SubgroupAny: {
-    return m_builder->CreateSubgroupAny(args[0]);
-  }
   case BuilderOpcode::SubgroupAllEqual: {
     return m_builder->CreateSubgroupAllEqual(args[0]);
   }