From f6a469d634a60c51b7bebbad9e7fe77937f78bb7 Mon Sep 17 00:00:00 2001 From: Aitor Camacho Date: Thu, 15 Aug 2024 10:01:46 +0900 Subject: [PATCH] WIP: VkDeviceMemory changes proposal Incomplete draft since it requires reworking how VkImage and VkBuffer handle the memory, but should be enough to provide an overview of the proposed changes. --- MoltenVK/MoltenVK/API/mvk_datatypes.h | 6 + .../MoltenVK/GPUObjects/MVKDeviceMemory.h | 122 +++--- .../MoltenVK/GPUObjects/MVKDeviceMemory.mm | 363 ++++++------------ MoltenVK/MoltenVK/GPUObjects/MVKImage.h | 2 +- MoltenVK/MoltenVK/Vulkan/mvk_datatypes.mm | 8 + 5 files changed, 199 insertions(+), 302 deletions(-) diff --git a/MoltenVK/MoltenVK/API/mvk_datatypes.h b/MoltenVK/MoltenVK/API/mvk_datatypes.h index 4d109957f..3acc1488e 100644 --- a/MoltenVK/MoltenVK/API/mvk_datatypes.h +++ b/MoltenVK/MoltenVK/API/mvk_datatypes.h @@ -488,6 +488,12 @@ MTLCPUCacheMode mvkMTLCPUCacheModeFromVkMemoryPropertyFlags(VkMemoryPropertyFlag /** Returns the Metal resource option flags corresponding to the Metal storage mode and cache mode. */ MTLResourceOptions mvkMTLResourceOptions(MTLStorageMode mtlStorageMode, MTLCPUCacheMode mtlCPUCacheMode); +/** Resturn the Metal storage mode from the Metal resource options */ +MTLStorageMode mvkMTLStorageMode(MTLResourceOptions options); + +/** Resturn the Metal cache mode from the Metal resource options */ +MTLCPUCacheMode mvkMTLCPUCacheMode(MTLResourceOptions options); + #ifdef __cplusplus } diff --git a/MoltenVK/MoltenVK/GPUObjects/MVKDeviceMemory.h b/MoltenVK/MoltenVK/GPUObjects/MVKDeviceMemory.h index 5449608a5..07f2fd4d2 100644 --- a/MoltenVK/MoltenVK/GPUObjects/MVKDeviceMemory.h +++ b/MoltenVK/MoltenVK/GPUObjects/MVKDeviceMemory.h @@ -33,17 +33,15 @@ static const VkExternalMemoryHandleTypeFlagBits VK_EXTERNAL_MEMORY_HANDLE_TYPE_M #pragma mark MVKDeviceMemory -typedef struct MVKMappedMemoryRange { - VkDeviceSize offset = 0; - VkDeviceSize size = 0; -} MVKMappedMemoryRange; - +typedef struct MVKMemoryRange { + VkDeviceSize offset = 0u; + VkDeviceSize size = 0u; +} MVKMemoryRange; /** Represents a Vulkan device-space memory allocation. */ class MVKDeviceMemory : public MVKVulkanAPIDeviceObject { public: - /** Returns the Vulkan type of this object. */ VkObjectType getVkObjectType() override { return VK_OBJECT_TYPE_DEVICE_MEMORY; } @@ -51,28 +49,29 @@ class MVKDeviceMemory : public MVKVulkanAPIDeviceObject { VkDebugReportObjectTypeEXT getVkDebugReportObjectType() override { return VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_MEMORY_EXT; } /** Returns whether the memory is accessible from the host. */ - inline bool isMemoryHostAccessible() { + inline bool isMemoryHostAccessible() { + MTLStorageMode storageMode = getMTLStorageMode(); #if MVK_APPLE_SILICON - if (_mtlStorageMode == MTLStorageModeMemoryless) - return false; + if (storageMode == MTLStorageModeMemoryless) + return false; #endif - return (_mtlStorageMode != MTLStorageModePrivate); - } + return (storageMode != MTLStorageModePrivate); + } /** Returns whether the memory is automatically coherent between device and host. */ - inline bool isMemoryHostCoherent() { return (_mtlStorageMode == MTLStorageModeShared); } + inline bool isMemoryHostCoherent() { return getMTLStorageMode() == MTLStorageModeShared; } - /** Returns whether this is a dedicated allocation. */ - inline bool isDedicatedAllocation() { return _isDedicated; } + /** Returns whether this is a dedicated allocation. */ + inline bool isDedicatedAllocation() { return _dedicatedResourceType != DedicatedResourceType::NONE; } - /** Returns the memory already committed by this instance. */ - inline VkDeviceSize getDeviceMemoryCommitment() { return _allocationSize; } + /** Returns the memory already committed by this instance. */ + inline VkDeviceSize getDeviceMemoryCommitment() { return _size; } /** * Returns the host memory address of this memory, or NULL if the memory has not been * mapped yet, or is marked as device-only and cannot be mapped to a host address. */ - inline void* getHostMemoryAddress() { return _pMemory; } + inline void* getHostMemoryAddress() { return _map; } /** * Maps the memory address at the specified offset from the start of this memory allocation, @@ -83,15 +82,8 @@ class MVKDeviceMemory : public MVKVulkanAPIDeviceObject { /** Unmaps a previously mapped memory range. */ VkResult unmap(const VkMemoryUnmapInfoKHR* unmapInfo); - /** - * If this device memory is currently mapped to host memory, returns the range within - * this device memory that is currently mapped to host memory, or returns {0,0} if - * this device memory is not currently mapped to host memory. - */ - inline const MVKMappedMemoryRange& getMappedRange() { return _mappedRange; } - /** Returns whether this device memory is currently mapped to host memory. */ - bool isMapped() { return _mappedRange.size > 0; } + bool isMapped() { return _map; } /** If this memory is host-visible, the specified memory range is flushed to the device. */ VkResult flushToDevice(VkDeviceSize offset, VkDeviceSize size); @@ -120,13 +112,13 @@ class MVKDeviceMemory : public MVKVulkanAPIDeviceObject { inline id getMTLHeap() { return _mtlHeap; } /** Returns the Metal storage mode used by this memory allocation. */ - inline MTLStorageMode getMTLStorageMode() { return _mtlStorageMode; } + inline MTLStorageMode getMTLStorageMode() { return mvkMTLStorageMode(_options); } /** Returns the Metal CPU cache mode used by this memory allocation. */ - inline MTLCPUCacheMode getMTLCPUCacheMode() { return _mtlCPUCacheMode; } + inline MTLCPUCacheMode getMTLCPUCacheMode() { return mvkMTLCPUCacheMode(_options); } /** Returns the Metal resource options used by this memory allocation. */ - inline MTLResourceOptions getMTLResourceOptions() { return mvkMTLResourceOptions(_mtlStorageMode, _mtlCPUCacheMode); } + inline MTLResourceOptions getMTLResourceOptions() { return _options; } #pragma mark Construction @@ -136,41 +128,61 @@ class MVKDeviceMemory : public MVKVulkanAPIDeviceObject { const VkMemoryAllocateInfo* pAllocateInfo, const VkAllocationCallbacks* pAllocator); - ~MVKDeviceMemory() override; + ~MVKDeviceMemory() override; protected: friend class MVKBuffer; - friend class MVKImageMemoryBinding; - friend class MVKImagePlane; + friend class MVKImageMemoryBinding; + friend class MVKImagePlane; void propagateDebugName() override; VkDeviceSize adjustMemorySize(VkDeviceSize size, VkDeviceSize offset); - VkResult addBuffer(MVKBuffer* mvkBuff); - void removeBuffer(MVKBuffer* mvkBuff); - VkResult addImageMemoryBinding(MVKImageMemoryBinding* mvkImg); - void removeImageMemoryBinding(MVKImageMemoryBinding* mvkImg); - bool ensureMTLHeap(); - bool ensureMTLBuffer(); - bool ensureHostMemory(); - void freeHostMemory(); - MVKResource* getDedicatedResource(); - void initExternalMemory(VkExternalMemoryHandleTypeFlags handleTypes); - - MVKSmallVector _buffers; - MVKSmallVector _imageMemoryBindings; - std::mutex _rezLock; - VkDeviceSize _allocationSize = 0; - MVKMappedMemoryRange _mappedRange; - id _mtlBuffer = nil; + void checkExternalMemoryRequirements(VkExternalMemoryHandleTypeFlags handleTypes); + + // Backing memory of VkDeviceMemory. This will not be allocated if memory was imported. + // Imported memory will directly be backed by MTLBuffer/MTLTexture since there's no way + // to create a MTLHeap with existing memory in Metal for now. id _mtlHeap = nil; - void* _pMemory = nullptr; - void* _pHostMemory = nullptr; + + // This MTLBuffer can have 3 usages: + // 1. When a heap is allocated, the buffer will extend the whole heap to be able to map and flush memory. + // 2. When there's no heap, the buffer will be the backing memory of VkDeviceMemory. + // 3. When a texture is imported, the GPU memory will be held by MTLTexture. However, if said texture is + // host accessible, we need to provide some memory for the mapping since Metal provides nothing. In this + // case, the buffer will hold the host memory that will later be copied to the texture once flushed. + id _mtlBuffer = nil; + + // If the user is importing a texture that is not backed by MTLHeap nor MTLBuffer, Metal does not expose + // anything to be able to access the texture data such as MTLBuffer::contents. This leads us to having to + // use the MTLTexture as the main GPU resource for the memory. If the texture is also host accessible, + // a buffer with host visible memory will be allocated as pointed out in point 3 before. + id _mtlTexture = nil; + + // Mapped memory. + void* _map = nullptr; + MVKMemoryRange _mapRange = { 0u, 0u }; + + // Allocation size. + VkDeviceSize _size = 0u; + // Metal resource options. + MTLResourceOptions _options = 0u; + + // When the allocation is dedicated, it will belong to one specific resource. + union { + MVKBuffer* _dedicatedBufferOwner = nullptr; + MVKImage* _dedicatedImageOwner; + }; + enum class DedicatedResourceType : uint8_t { + NONE = 0, + BUFFER, + IMAGE + }; + DedicatedResourceType _dedicatedResourceType = DedicatedResourceType::NONE; + VkMemoryPropertyFlags _vkMemPropFlags; - VkMemoryAllocateFlags _vkMemAllocFlags; - MTLStorageMode _mtlStorageMode; - MTLCPUCacheMode _mtlCPUCacheMode; - bool _isDedicated = false; - bool _isHostMemImported = false; + // Tracks if we need to flush from MTLBuffer to MTLTexture. Used only when memory is an imported texture + // that had no backing MTLBuffer nor MTLHeap + bool _requiresFlushingBufferToTexture = false; }; diff --git a/MoltenVK/MoltenVK/GPUObjects/MVKDeviceMemory.mm b/MoltenVK/MoltenVK/GPUObjects/MVKDeviceMemory.mm index 1ec518488..17865b124 100644 --- a/MoltenVK/MoltenVK/GPUObjects/MVKDeviceMemory.mm +++ b/MoltenVK/MoltenVK/GPUObjects/MVKDeviceMemory.mm @@ -33,6 +33,7 @@ void MVKDeviceMemory::propagateDebugName() { setLabelIfNotNil(_mtlHeap, _debugName); setLabelIfNotNil(_mtlBuffer, _debugName); + setLabelIfNotNil(_mtlTexture, _debugName); } VkResult MVKDeviceMemory::map(const VkMemoryMapInfoKHR* pMemoryMapInfo, void** ppData) { @@ -44,14 +45,11 @@ return reportError(VK_ERROR_MEMORY_MAP_FAILED, "Memory is already mapped. Call vkUnmapMemory() first."); } - if ( !ensureMTLBuffer() && !ensureHostMemory() ) { - return reportError(VK_ERROR_OUT_OF_HOST_MEMORY, "Could not allocate %llu bytes of host-accessible device memory.", _allocationSize); - } - - _mappedRange.offset = pMemoryMapInfo->offset; - _mappedRange.size = adjustMemorySize(pMemoryMapInfo->size, pMemoryMapInfo->offset); + _mapRange.offset = pMemoryMapInfo->offset; + _mapRange.size = adjustMemorySize(pMemoryMapInfo->size, pMemoryMapInfo->offset); + _map = [_mtlBuffer contents]; - *ppData = (void*)((uintptr_t)_pMemory + pMemoryMapInfo->offset); + *ppData = (void*)((uintptr_t)_map + pMemoryMapInfo->offset); // Coherent memory does not require flushing by app, so we must flush now // to support Metal textures that actually reside in non-coherent memory. @@ -70,11 +68,11 @@ // Coherent memory does not require flushing by app, so we must flush now // to support Metal textures that actually reside in non-coherent memory. if (mvkIsAnyFlagEnabled(_vkMemPropFlags, VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) { - flushToDevice(_mappedRange.offset, _mappedRange.size); + flushToDevice(_mapRange.offset, _mapRange.size); } - _mappedRange.offset = 0; - _mappedRange.size = 0; + _mapRange.offset = 0; + _mapRange.size = 0; return VK_SUCCESS; } @@ -83,224 +81,74 @@ VkDeviceSize memSize = adjustMemorySize(size, offset); if (memSize == 0 || !isMemoryHostAccessible()) { return VK_SUCCESS; } + if (_requiresFlushingBufferToTexture) { + // TODO Aitor: Flush buffer to texture + } + #if MVK_MACOS - if ( !isUnifiedMemoryGPU() && _mtlBuffer && _mtlStorageMode == MTLStorageModeManaged) { + if ( !isUnifiedMemoryGPU() && _mtlBuffer && getMTLStorageMode() == MTLStorageModeManaged) { [_mtlBuffer didModifyRange: NSMakeRange(offset, memSize)]; } #endif - // If we have an MTLHeap object, there's no need to sync memory manually between resources and the buffer. - if ( !_mtlHeap ) { - lock_guard lock(_rezLock); - for (auto& img : _imageMemoryBindings) { img->flushToDevice(offset, memSize); } - for (auto& buf : _buffers) { buf->flushToDevice(offset, memSize); } - } - return VK_SUCCESS; } VkResult MVKDeviceMemory::pullFromDevice(VkDeviceSize offset, VkDeviceSize size, MVKMTLBlitEncoder* pBlitEnc) { - VkDeviceSize memSize = adjustMemorySize(size, offset); + VkDeviceSize memSize = adjustMemorySize(size, offset); if (memSize == 0 || !isMemoryHostAccessible()) { return VK_SUCCESS; } + + MTLStorageMode storageMode = getMTLStorageMode(); + + if (_requiresFlushingBufferToTexture) { + // TODO Aitor: Flush texture to buffer + } #if MVK_MACOS - if ( !isUnifiedMemoryGPU() && pBlitEnc && _mtlBuffer && _mtlStorageMode == MTLStorageModeManaged) { + if ( !isUnifiedMemoryGPU() && pBlitEnc && _mtlBuffer && storageMode == MTLStorageModeManaged) { if ( !pBlitEnc->mtlCmdBuffer) { pBlitEnc->mtlCmdBuffer = _device->getAnyQueue()->getMTLCommandBuffer(kMVKCommandUseInvalidateMappedMemoryRanges); } if ( !pBlitEnc->mtlBlitEncoder) { pBlitEnc->mtlBlitEncoder = [pBlitEnc->mtlCmdBuffer blitCommandEncoder]; } [pBlitEnc->mtlBlitEncoder synchronizeResource: _mtlBuffer]; } #endif - // If we have an MTLHeap object, there's no need to sync memory manually between resources and the buffer. - if ( !_mtlHeap ) { - lock_guard lock(_rezLock); - for (auto& img : _imageMemoryBindings) { img->pullFromDevice(offset, memSize); } - for (auto& buf : _buffers) { buf->pullFromDevice(offset, memSize); } - } - return VK_SUCCESS; } // If the size parameter is the special constant VK_WHOLE_SIZE, returns the size of memory // between offset and the end of the buffer, otherwise simply returns size. VkDeviceSize MVKDeviceMemory::adjustMemorySize(VkDeviceSize size, VkDeviceSize offset) { - return (size == VK_WHOLE_SIZE) ? (_allocationSize - offset) : size; -} - -VkResult MVKDeviceMemory::addBuffer(MVKBuffer* mvkBuff) { - lock_guard lock(_rezLock); - - // If a dedicated alloc, ensure this buffer is the one and only buffer - // I am dedicated to. - if (_isDedicated && (_buffers.empty() || _buffers[0] != mvkBuff) ) { - return reportError(VK_ERROR_OUT_OF_DEVICE_MEMORY, "Could not bind VkBuffer %p to a VkDeviceMemory dedicated to resource %p. A dedicated allocation may only be used with the resource it was dedicated to.", mvkBuff, getDedicatedResource() ); - } - - if (!ensureMTLBuffer() ) { - return reportError(VK_ERROR_OUT_OF_DEVICE_MEMORY, "Could not bind a VkBuffer to a VkDeviceMemory of size %llu bytes. The maximum memory-aligned size of a VkDeviceMemory that supports a VkBuffer is %llu bytes.", _allocationSize, getMetalFeatures().maxMTLBufferSize); - } - - // In the dedicated case, we already saved the buffer we're going to use. - if (!_isDedicated) { _buffers.push_back(mvkBuff); } - - return VK_SUCCESS; -} - -void MVKDeviceMemory::removeBuffer(MVKBuffer* mvkBuff) { - lock_guard lock(_rezLock); - mvkRemoveAllOccurances(_buffers, mvkBuff); -} - -VkResult MVKDeviceMemory::addImageMemoryBinding(MVKImageMemoryBinding* mvkImg) { - lock_guard lock(_rezLock); - - // If a dedicated alloc, ensure this image is the one and only image - // I am dedicated to. If my image is aliasable, though, allow other aliasable - // images to bind to me. - if (_isDedicated && (_imageMemoryBindings.empty() || !(mvkContains(_imageMemoryBindings, mvkImg) || (_imageMemoryBindings[0]->_image->getIsAliasable() && mvkImg->_image->getIsAliasable()))) ) { - return reportError(VK_ERROR_OUT_OF_DEVICE_MEMORY, "Could not bind VkImage %p to a VkDeviceMemory dedicated to resource %p. A dedicated allocation may only be used with the resource it was dedicated to.", mvkImg, getDedicatedResource() ); - } - - if (!_isDedicated) { _imageMemoryBindings.push_back(mvkImg); } - - return VK_SUCCESS; -} - -void MVKDeviceMemory::removeImageMemoryBinding(MVKImageMemoryBinding* mvkImg) { - lock_guard lock(_rezLock); - mvkRemoveAllOccurances(_imageMemoryBindings, mvkImg); -} - -// Ensures that this instance is backed by a MTLHeap object, -// creating the MTLHeap if needed, and returns whether it was successful. -bool MVKDeviceMemory::ensureMTLHeap() { - - if (_mtlHeap) { return true; } - - // Can't create a MTLHeap on imported memory - if (_isHostMemImported) { return true; } - - // Don't bother if we don't have placement heaps. - if (!getMetalFeatures().placementHeaps) { return true; } - - // Can't create MTLHeaps of zero size. - if (_allocationSize == 0) { return true; } - -#if MVK_MACOS - // MTLHeaps on macOS must use private storage for now. - if (_mtlStorageMode != MTLStorageModePrivate) { return true; } -#endif -#if MVK_IOS - // MTLHeaps on iOS must use private or shared storage for now. - if ( !(_mtlStorageMode == MTLStorageModePrivate || - _mtlStorageMode == MTLStorageModeShared) ) { return true; } -#endif - - MTLHeapDescriptor* heapDesc = [MTLHeapDescriptor new]; - heapDesc.type = MTLHeapTypePlacement; - heapDesc.storageMode = _mtlStorageMode; - heapDesc.cpuCacheMode = _mtlCPUCacheMode; - // For now, use tracked resources. Later, we should probably default - // to untracked, since Vulkan uses explicit barriers anyway. - heapDesc.hazardTrackingMode = MTLHazardTrackingModeTracked; - heapDesc.size = _allocationSize; - _mtlHeap = [getMTLDevice() newHeapWithDescriptor: heapDesc]; // retained - [heapDesc release]; - if (!_mtlHeap) { return false; } - - propagateDebugName(); - - return true; -} - -// Ensures that this instance is backed by a MTLBuffer object, -// creating the MTLBuffer if needed, and returns whether it was successful. -bool MVKDeviceMemory::ensureMTLBuffer() { - - if (_mtlBuffer) { return true; } - - NSUInteger memLen = mvkAlignByteCount(_allocationSize, getMetalFeatures().mtlBufferAlignment); - - if (memLen > getMetalFeatures().maxMTLBufferSize) { return false; } - - // If host memory was already allocated, it is copied into the new MTLBuffer, and then released. - if (_mtlHeap) { - _mtlBuffer = [_mtlHeap newBufferWithLength: memLen options: getMTLResourceOptions() offset: 0]; // retained - if (_pHostMemory) { - memcpy(_mtlBuffer.contents, _pHostMemory, memLen); - freeHostMemory(); - } - [_mtlBuffer makeAliasable]; - } else if (_pHostMemory) { - auto rezOpts = getMTLResourceOptions(); - if (_isHostMemImported) { - _mtlBuffer = [getMTLDevice() newBufferWithBytesNoCopy: _pHostMemory length: memLen options: rezOpts deallocator: nil]; // retained - } else { - _mtlBuffer = [getMTLDevice() newBufferWithBytes: _pHostMemory length: memLen options: rezOpts]; // retained - } - freeHostMemory(); - } else { - _mtlBuffer = [getMTLDevice() newBufferWithLength: memLen options: getMTLResourceOptions()]; // retained - } - if (!_mtlBuffer) { return false; } - _pMemory = isMemoryHostAccessible() ? _mtlBuffer.contents : nullptr; - - propagateDebugName(); - - return true; -} - -// Ensures that host-accessible memory is available, allocating it if necessary. -bool MVKDeviceMemory::ensureHostMemory() { - - if (_pMemory) { return true; } - - if ( !_pHostMemory) { - size_t memAlign = getMetalFeatures().mtlBufferAlignment; - NSUInteger memLen = mvkAlignByteCount(_allocationSize, memAlign); - int err = posix_memalign(&_pHostMemory, memAlign, memLen); - if (err) { return false; } - } - - _pMemory = _pHostMemory; - - return true; -} - -void MVKDeviceMemory::freeHostMemory() { - if ( !_isHostMemImported ) { free(_pHostMemory); } - _pHostMemory = nullptr; -} - -MVKResource* MVKDeviceMemory::getDedicatedResource() { - MVKAssert(_isDedicated, "This method should only be called on dedicated allocations!"); - return _buffers.empty() ? (MVKResource*)_imageMemoryBindings[0] : (MVKResource*)_buffers[0]; + return (size == VK_WHOLE_SIZE) ? (_size - offset) : size; } MVKDeviceMemory::MVKDeviceMemory(MVKDevice* device, const VkMemoryAllocateInfo* pAllocateInfo, const VkAllocationCallbacks* pAllocator) : MVKVulkanAPIDeviceObject(device) { - // Set Metal memory parameters - _vkMemAllocFlags = 0; - _vkMemPropFlags = getDeviceMemoryProperties().memoryTypes[pAllocateInfo->memoryTypeIndex].propertyFlags; - _mtlStorageMode = getPhysicalDevice()->getMTLStorageModeFromVkMemoryPropertyFlags(_vkMemPropFlags); - _mtlCPUCacheMode = mvkMTLCPUCacheModeFromVkMemoryPropertyFlags(_vkMemPropFlags); + MVKPhysicalDevice* physicalDevice = getPhysicalDevice(); + id mtlDevice = physicalDevice->getMTLDevice(); - _allocationSize = pAllocateInfo->allocationSize; + // Set Metal memory parameters + _vkMemPropFlags = physicalDevice->getMemoryProperties()->memoryTypes[pAllocateInfo->memoryTypeIndex].propertyFlags; + MTLStorageMode storageMode = physicalDevice->getMTLStorageModeFromVkMemoryPropertyFlags(_vkMemPropFlags); + MTLCPUCacheMode cpuCacheMode = mvkMTLCPUCacheModeFromVkMemoryPropertyFlags(_vkMemPropFlags); + _options = mvkMTLResourceOptions(storageMode, cpuCacheMode); + _size = pAllocateInfo->allocationSize; bool willExportMTLBuffer = false; - VkImage dedicatedImage = VK_NULL_HANDLE; - VkBuffer dedicatedBuffer = VK_NULL_HANDLE; VkExternalMemoryHandleTypeFlags handleTypes = 0; for (const auto* next = (const VkBaseInStructure*)pAllocateInfo->pNext; next; next = next->pNext) { switch (next->sType) { case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO: { auto* pDedicatedInfo = (VkMemoryDedicatedAllocateInfo*)next; - dedicatedImage = pDedicatedInfo->image; - dedicatedBuffer = pDedicatedInfo->buffer; - _isDedicated = dedicatedImage || dedicatedBuffer; + if (pDedicatedInfo->image) { + _dedicatedImageOwner = reinterpret_cast(pDedicatedInfo->image); + _dedicatedResourceType = DedicatedResourceType::IMAGE; + } else if (pDedicatedInfo->buffer) { + _dedicatedBufferOwner = reinterpret_cast(pDedicatedInfo->buffer); + _dedicatedResourceType = DedicatedResourceType::BUFFER; + } break; } case VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT: { @@ -309,14 +157,15 @@ switch (pMemHostPtrInfo->handleType) { case VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT: case VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_MAPPED_FOREIGN_MEMORY_BIT_EXT: - _pHostMemory = pMemHostPtrInfo->pHostPointer; - _isHostMemImported = true; + // Since there's no way to allocate a heap using external host memory, we default to a buffer + _mtlBuffer = [mtlDevice newBufferWithBytesNoCopy: pMemHostPtrInfo->pHostPointer length: _size options: _options deallocator: ^(void *pointer, NSUInteger length){ free(pointer); }]; break; default: break; } } else { setConfigurationResult(reportError(VK_ERROR_INVALID_EXTERNAL_HANDLE_KHR, "vkAllocateMemory(): Imported memory must be host-visible.")); + return; } break; } @@ -328,75 +177,91 @@ case VK_STRUCTURE_TYPE_IMPORT_METAL_BUFFER_INFO_EXT: { // Setting Metal objects directly will override Vulkan settings. // It is responsibility of app to ensure these are consistent. Not doing so results in undefined behavior. - const auto* pMTLBuffInfo = (VkImportMetalBufferInfoEXT*)next; - [_mtlBuffer release]; // guard against dups - _mtlBuffer = [pMTLBuffInfo->mtlBuffer retain]; // retained - _mtlStorageMode = _mtlBuffer.storageMode; - _mtlCPUCacheMode = _mtlBuffer.cpuCacheMode; - _allocationSize = _mtlBuffer.length; + const auto* pMTLBufferInfo = (VkImportMetalBufferInfoEXT*)next; + _mtlBuffer = [pMTLBufferInfo->mtlBuffer retain]; // retained + _options = _mtlBuffer.resourceOptions; + _size = _mtlBuffer.length; + if (_mtlBuffer.heap) + _mtlHeap = [_mtlBuffer.heap retain]; break; } - case VK_STRUCTURE_TYPE_EXPORT_METAL_OBJECT_CREATE_INFO_EXT: { - const auto* pExportInfo = (VkExportMetalObjectCreateInfoEXT*)next; - willExportMTLBuffer = pExportInfo->exportObjectType == VK_EXPORT_METAL_OBJECT_TYPE_METAL_BUFFER_BIT_EXT; - } - case VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO: { - auto* pMemAllocFlagsInfo = (VkMemoryAllocateFlagsInfo*)next; - _vkMemAllocFlags = pMemAllocFlagsInfo->flags; - break; + case VK_STRUCTURE_TYPE_IMPORT_METAL_TEXTURE_INFO_EXT: { + const auto* pMTLTextureInfo = (VkImportMetalTextureInfoEXT*)next; + _mtlTexture = [pMTLTextureInfo->mtlTexture retain]; + // Imported host visible textures require a way to map them. Since Metal does not provide any + // utility to access texture data from a MTLTexture like MTLBuffer::contents. + if (_mtlTexture.heap && _mtlTexture.heap.type == MTLHeapTypePlacement) { + _mtlHeap = [_mtlTexture.heap retain]; + if (_mtlTexture.buffer) { + _mtlBuffer = [_mtlTexture.buffer retain]; + } else { + _mtlBuffer = [_mtlHeap newBufferWithLength:_size options:_options offset:_mtlTexture.heapOffset]; + } + } else { + if (_mtlTexture.buffer) { + _mtlBuffer = [_mtlTexture.buffer retain]; + } else { + void* data = malloc(_size); + _mtlBuffer = [mtlDevice newBufferWithBytesNoCopy: data length: _size options: _options deallocator: ^(void *pointer, NSUInteger length){ free(pointer); }]; + _requiresFlushingBufferToTexture = true; + } + } } default: break; } } - initExternalMemory(handleTypes); // After setting _isDedicated + // Once we know the type of external handle + checkExternalMemoryRequirements(handleTypes); // "Dedicated" means this memory can only be used for this image or buffer. - if (dedicatedImage) { + if (_dedicatedResourceType == DedicatedResourceType::IMAGE) { #if MVK_MACOS if (isMemoryHostCoherent() ) { - if (!((MVKImage*)dedicatedImage)->_isLinear) { + if (!_dedicatedImageOwner->_isLinear) { setConfigurationResult(reportError(VK_ERROR_OUT_OF_DEVICE_MEMORY, "vkAllocateMemory(): Host-coherent VkDeviceMemory objects cannot be associated with optimal-tiling images.")); + return; } else { if (!getMetalFeatures().sharedLinearTextures) { // Need to use the managed mode for images. - _mtlStorageMode = MTLStorageModeManaged; - } - // Nonetheless, we need a buffer to be able to map the memory at will. - if (!ensureMTLBuffer() ) { - setConfigurationResult(reportError(VK_ERROR_OUT_OF_DEVICE_MEMORY, "vkAllocateMemory(): Could not allocate a host-coherent VkDeviceMemory of size %llu bytes. The maximum memory-aligned size of a host-coherent VkDeviceMemory is %llu bytes.", _allocationSize, getMetalFeatures().maxMTLBufferSize)); + storageMode = MTLStorageModeManaged; + _options = mvkMTLResourceOptions(storageMode, cpuCacheMode); } } } #endif - for (auto& memoryBinding : ((MVKImage*)dedicatedImage)->_memoryBindings) { - _imageMemoryBindings.push_back(memoryBinding); - } - return; } - if (dedicatedBuffer) { - _buffers.push_back((MVKBuffer*)dedicatedBuffer); - } + // Only allocate memory if it was not imported + if (_mtlBuffer) + return; - // If we can, create a MTLHeap. This should happen before creating the buffer, allowing us to map its contents. - if ( !_isDedicated ) { - if (!ensureMTLHeap()) { - setConfigurationResult(reportError(VK_ERROR_OUT_OF_DEVICE_MEMORY, "vkAllocateMemory(): Could not allocate VkDeviceMemory of size %llu bytes.", _allocationSize)); - return; - } - } + MTLHeapDescriptor* heapDesc = [MTLHeapDescriptor new]; + heapDesc.type = MTLHeapTypePlacement; + heapDesc.resourceOptions = _options; + // For now, use tracked resources. Later, we should probably default + // to untracked, since Vulkan uses explicit barriers anyway. + heapDesc.hazardTrackingMode = MTLHazardTrackingModeTracked; + heapDesc.size = _size; + _mtlHeap = [mtlDevice newHeapWithDescriptor: heapDesc]; // retained + [heapDesc release]; + if (!_mtlHeap) goto fail_alloc; + propagateDebugName(); - // If memory needs to be coherent it must reside in a MTLBuffer, since an open-ended map() must work. - // If memory was imported, a MTLBuffer must be created on it. - // Or if a MTLBuffer will be exported, ensure it exists. - if ((isMemoryHostCoherent() || _isHostMemImported || willExportMTLBuffer) && !ensureMTLBuffer() ) { - setConfigurationResult(reportError(VK_ERROR_OUT_OF_DEVICE_MEMORY, "vkAllocateMemory(): Could not allocate a host-coherent or exportable VkDeviceMemory of size %llu bytes. The maximum memory-aligned size of a host-coherent VkDeviceMemory is %llu bytes.", _allocationSize, getMetalFeatures().maxMTLBufferSize)); + // Create a buffer that expands the whole heap + // to be able to map and flush as required by the application + if (mvkIsAnyFlagEnabled(_vkMemPropFlags, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)) { + _mtlBuffer = [_mtlHeap newBufferWithLength:_size options:_options]; + if (!_mtlBuffer) goto fail_alloc; + [_mtlBuffer makeAliasable]; } + +fail_alloc: + setConfigurationResult(reportError(VK_ERROR_OUT_OF_DEVICE_MEMORY, "vkAllocateMemory(): Could not allocate VkDeviceMemory of size %llu bytes.", _size)); } -void MVKDeviceMemory::initExternalMemory(VkExternalMemoryHandleTypeFlags handleTypes) { +void MVKDeviceMemory::checkExternalMemoryRequirements(VkExternalMemoryHandleTypeFlags handleTypes) { if ( !handleTypes ) { return; } if ( !mvkIsOnlyAnyFlagEnabled(handleTypes, VK_EXTERNAL_MEMORY_HANDLE_TYPE_MTLBUFFER_BIT_KHR | VK_EXTERNAL_MEMORY_HANDLE_TYPE_MTLTEXTURE_BIT_KHR) ) { @@ -412,24 +277,30 @@ auto& xmProps = getPhysicalDevice()->getExternalImageProperties(VK_EXTERNAL_MEMORY_HANDLE_TYPE_MTLTEXTURE_BIT_KHR); requiresDedicated = requiresDedicated || mvkIsAnyFlagEnabled(xmProps.externalMemoryFeatures, VK_EXTERNAL_MEMORY_FEATURE_DEDICATED_ONLY_BIT); } - if (requiresDedicated && !_isDedicated) { + if (requiresDedicated && (_dedicatedResourceType == DedicatedResourceType::NONE)) { setConfigurationResult(reportError(VK_ERROR_INITIALIZATION_FAILED, "vkAllocateMemory(): External memory requires a dedicated VkBuffer or VkImage.")); } } MVKDeviceMemory::~MVKDeviceMemory() { - // Unbind any resources that are using me. Iterate a copy of the collection, - // to allow the resource to callback to remove itself from the collection. - auto buffCopies = _buffers; - for (auto& buf : buffCopies) { buf->bindDeviceMemory(nullptr, 0); } - auto imgCopies = _imageMemoryBindings; - for (auto& img : imgCopies) { img->bindDeviceMemory(nullptr, 0); } - - [_mtlBuffer release]; - _mtlBuffer = nil; + if (_mtlTexture) { + [_mtlTexture release]; + _mtlTexture = nil; + + // Having no buffer and texture being host accessible means we allocated memory for the mapping + if (!_mtlBuffer && mvkIsAnyFlagEnabled(_vkMemPropFlags, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)) { + free(_map); + _map = nullptr; + } + } - [_mtlHeap release]; - _mtlHeap = nil; + if (_mtlBuffer) { + [_mtlBuffer release]; + _mtlBuffer = nil; + } - freeHostMemory(); + if (_mtlHeap) { + [_mtlHeap release]; + _mtlHeap = nil; + } } diff --git a/MoltenVK/MoltenVK/GPUObjects/MVKImage.h b/MoltenVK/MoltenVK/GPUObjects/MVKImage.h index e42150838..e09e620f5 100644 --- a/MoltenVK/MoltenVK/GPUObjects/MVKImage.h +++ b/MoltenVK/MoltenVK/GPUObjects/MVKImage.h @@ -79,7 +79,7 @@ class MVKImagePlane : public MVKBaseObject { MVKCommandUse cmdUse); void pullFromDeviceOnCompletion(MVKCommandEncoder* cmdEncoder, MVKImageSubresource& subresource, - const MVKMappedMemoryRange& mappedRange); + const MVKMemoryRange& mapRange); MVKImagePlane(MVKImage* image, uint8_t planeIndex); diff --git a/MoltenVK/MoltenVK/Vulkan/mvk_datatypes.mm b/MoltenVK/MoltenVK/Vulkan/mvk_datatypes.mm index 961e6e73f..a33102c44 100644 --- a/MoltenVK/MoltenVK/Vulkan/mvk_datatypes.mm +++ b/MoltenVK/MoltenVK/Vulkan/mvk_datatypes.mm @@ -895,3 +895,11 @@ MVK_PUBLIC_SYMBOL MTLResourceOptions mvkMTLResourceOptions(MTLStorageMode mtlSto MTLCPUCacheMode mtlCPUCacheMode) { return (mtlStorageMode << MTLResourceStorageModeShift) | (mtlCPUCacheMode << MTLResourceCPUCacheModeShift); } + +MVK_PUBLIC_SYMBOL MTLStorageMode mvkMTLStorageMode(MTLResourceOptions options) { + return static_cast((options & MTLResourceStorageModeMask) >> MTLResourceStorageModeShift); +} + +MVK_PUBLIC_SYMBOL MTLCPUCacheMode mvkMTLCPUCacheMode(MTLResourceOptions options) { + return static_cast((options & MTLResourceCPUCacheModeMask) >> MTLResourceCPUCacheModeShift); +}