Fix Z-test and Z-write being enabled by default for main queue (axmol…

…engine#1981)

* Backend: remove RenderTargetFlag and refactor depth/stencil state setup

core/2d/Grid.cpp

@@ -219,7 +219,7 @@ void GridBase::beforeDraw()
         _oldRenderTarget = renderer->getRenderTarget();
         AX_SAFE_RELEASE(_renderTarget);
         _renderTarget =
-            backend::DriverBase::getInstance()->newRenderTarget(TargetBufferFlags::COLOR, _texture->getBackendTexture());
+            backend::DriverBase::getInstance()->newRenderTarget(_texture->getBackendTexture());
         renderer->setRenderTarget(_renderTarget);
     };
     renderer->addCallbackCommand(beforeDrawCommandFunc);

core/2d/RenderTexture.cpp

@@ -187,11 +187,9 @@ bool RenderTexture::initWithWidthAndHeight(int w,
         descriptor.textureFormat = PixelFormat::RGBA8;
         _texture2D               = new Texture2D();
         _texture2D->updateTextureDescriptor(descriptor, !!AX_ENABLE_PREMULTIPLIED_ALPHA);
-        _renderTargetFlags = RenderTargetFlag::COLOR;
 
         if (PixelFormat::D24S8 == depthStencilFormat || sharedRenderTarget)
         {
-            _renderTargetFlags       = RenderTargetFlag::ALL;
             descriptor.textureFormat = PixelFormat::D24S8;
 
             AX_SAFE_RELEASE(_depthStencilTexture);
@@ -210,7 +208,7 @@ bool RenderTexture::initWithWidthAndHeight(int w,
         else
         {
              _renderTarget = backend::DriverBase::getInstance()->newRenderTarget(
-                 _renderTargetFlags, _texture2D ? _texture2D->getBackendTexture() : nullptr,
+                 _texture2D ? _texture2D->getBackendTexture() : nullptr,
                  _depthStencilTexture ? _depthStencilTexture->getBackendTexture() : nullptr,
                  _depthStencilTexture ? _depthStencilTexture->getBackendTexture() : nullptr);	        
         }

core/2d/RenderTexture.h

@@ -396,7 +396,6 @@ class AX_DLL RenderTexture : public Node
 
     backend::RenderTarget* _renderTarget    = nullptr;
     backend::RenderTarget* _oldRenderTarget = nullptr;
-    RenderTargetFlag _renderTargetFlags{};
 
     RefPtr<Image> _UITextureImage            = nullptr;
     backend::PixelFormat _pixelFormat = backend::PixelFormat::RGBA8;

core/base/Types.h

@@ -469,7 +469,6 @@ using PixelFormat  = backend::PixelFormat;
 
 using TargetBufferFlags = backend::TargetBufferFlags;
 using DepthStencilFlags = backend::DepthStencilFlags;
-using RenderTargetFlag  = backend::RenderTargetFlag;
 using ClearFlag         = backend::ClearFlag;
 
 typedef void (*AsyncOperation)(void* param);

core/renderer/Renderer.cpp

@@ -204,9 +204,8 @@ void Renderer::init()
 
     auto driver    = backend::DriverBase::getInstance();
     _commandBuffer = driver->newCommandBuffer();
-    // @MTL: the depth stencil flags must same render target and _dsDesc
     _dsDesc.flags = DepthStencilFlags::ALL;
-    _defaultRT    = driver->newDefaultRenderTarget(TargetBufferFlags::COLOR | TargetBufferFlags::DEPTH_AND_STENCIL);
+    _defaultRT    = driver->newDefaultRenderTarget();
 
     _currentRT      = _defaultRT;
     _renderPipeline = driver->newRenderPipeline();
@@ -218,7 +217,7 @@ void Renderer::init()
 
 backend::RenderTarget* Renderer::getOffscreenRenderTarget() {
     if (_offscreenRT != nullptr) return _offscreenRT;
-    return (_offscreenRT = backend::DriverBase::getInstance()->newRenderTarget(TargetBufferFlags::COLOR | TargetBufferFlags::DEPTH_AND_STENCIL));
+    return (_offscreenRT = backend::DriverBase::getInstance()->newRenderTarget());
 }
 
 void Renderer::addCallbackCommand(std::function<void()> func, float globalZOrder)
@@ -283,13 +282,7 @@ void Renderer::processGroupCommand(GroupCommand* command)
 
     int renderQueueID = ((GroupCommand*)command)->getRenderQueueID();
 
-    pushStateBlock();
-    // apply default state for all render queues
-    setDepthTest(false);
-    setDepthWrite(false);
-    setCullMode(backend::CullMode::NONE);
     visitRenderQueue(_renderGroups[renderQueueID]);
-    popStateBlock();
 }
 
 void Renderer::processRenderCommand(RenderCommand* command)
@@ -358,6 +351,13 @@ void Renderer::processRenderCommand(RenderCommand* command)
 
 void Renderer::visitRenderQueue(RenderQueue& queue)
 {
+    pushStateBlock();
+
+    // Apply default state for all render queues
+    setDepthTest(false);
+    setDepthWrite(false);
+    setCullMode(backend::CullMode::NONE);
+
     //
     // Process Global-Z < 0 Objects
     //
@@ -388,6 +388,8 @@ void Renderer::visitRenderQueue(RenderQueue& queue)
     // Process Global-Z > 0 Queue
     //
     doVisitRenderQueue(queue.getSubQueue(RenderQueue::QUEUE_GROUP::GLOBALZ_POS));
+
+    popStateBlock();
 }
 
 void Renderer::doVisitRenderQueue(const std::vector<RenderCommand*>& renderCommands)
@@ -455,29 +457,17 @@ void Renderer::clean()
 void Renderer::setDepthTest(bool value)
 {
     if (value)
-    {
-        _currentRT->addFlag(TargetBufferFlags::DEPTH);
         _dsDesc.addFlag(DepthStencilFlags::DEPTH_TEST);
-    }
     else
-    {
-        _currentRT->removeFlag(TargetBufferFlags::DEPTH);
         _dsDesc.removeFlag(DepthStencilFlags::DEPTH_TEST);
-    }
 }
 
 void Renderer::setStencilTest(bool value)
 {
     if (value)
-    {
-        _currentRT->addFlag(TargetBufferFlags::STENCIL);
         _dsDesc.addFlag(DepthStencilFlags::STENCIL_TEST);
-    }
     else
-    {
-        _currentRT->removeFlag(TargetBufferFlags::STENCIL);
         _dsDesc.removeFlag(DepthStencilFlags::STENCIL_TEST);
-    }
 }
 
 void Renderer::setDepthWrite(bool value)
@@ -849,7 +839,18 @@ void Renderer::readPixels(backend::RenderTarget* rt,
 void Renderer::beginRenderPass()
 {
     _commandBuffer->beginRenderPass(_currentRT, _renderPassDesc);
-    _commandBuffer->updateDepthStencilState(_dsDesc);
+
+    // Disable depth/stencil access if render target has no relevant attachments.
+    auto depthStencil = _dsDesc;
+    if (!_currentRT->isDefaultRenderTarget())
+    {
+        if (!_currentRT->_depth)
+            depthStencil.removeFlag(DepthStencilFlags::DEPTH_TEST | DepthStencilFlags::DEPTH_WRITE);
+        if (!_currentRT->_stencil)
+            depthStencil.removeFlag(DepthStencilFlags::STENCIL_TEST);
+    }
+
+    _commandBuffer->updateDepthStencilState(depthStencil);
     _commandBuffer->setStencilReferenceValue(_stencilRef);
 
     _commandBuffer->setViewport(_viewport.x, _viewport.y, _viewport.width, _viewport.height);
@@ -929,11 +930,6 @@ ClearFlag Renderer::getClearFlag() const
     return _clearFlag;
 }
 
-RenderTargetFlag Renderer::getRenderTargetFlag() const
-{
-    return _currentRT->getTargetFlags();
-}
-
 void Renderer::setScissorTest(bool enabled)
 {
     _scissorState.isEnabled = enabled;

core/renderer/Renderer.h

@@ -2,7 +2,7 @@
  Copyright (c) 2013-2016 Chukong Technologies Inc.
  Copyright (c) 2017-2018 Xiamen Yaji Software Co., Ltd.
  Copyright (c) 2019-present Axmol Engine contributors (see AUTHORS.md).
-  
+
  https://axmol.dev/
 
  Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -234,12 +234,6 @@ class AX_DLL Renderer
      */
     ClearFlag getClearFlag() const;
 
-    /**
-     * Get the render target flag.
-     * @return The render target flag.
-     */
-    RenderTargetFlag getRenderTargetFlag() const;
-
     // depth/stencil state.
 
     /**

core/renderer/backend/DepthStencilState.h

@@ -62,7 +62,6 @@ struct DepthStencilDescriptor
     StencilDescriptor frontFaceStencil;
     void addFlag(DepthStencilFlags flag) { this->flags |= flag; }
     void removeFlag(DepthStencilFlags flag) { this->flags &= ~flag; }
-    // must match current render target
     DepthStencilFlags flags = DepthStencilFlags::ALL;
 };
 

core/renderer/backend/DriverBase.h

@@ -110,10 +110,9 @@ class AX_DLL DriverBase
      */
     virtual TextureBackend* newTexture(const TextureDescriptor& descriptor) = 0;
 
-    virtual RenderTarget* newDefaultRenderTarget(TargetBufferFlags rtf) = 0;
+    virtual RenderTarget* newDefaultRenderTarget() = 0;
 
-    virtual RenderTarget* newRenderTarget(TargetBufferFlags rtf,
-                                          TextureBackend* colorAttachment    = nullptr,
+    virtual RenderTarget* newRenderTarget(TextureBackend* colorAttachment    = nullptr,
                                           TextureBackend* depthAttachment    = nullptr,
                                           TextureBackend* stencilAttachhment = nullptr) = 0;
 

core/renderer/backend/RenderTarget.h

@@ -28,18 +28,6 @@ class RenderTarget : public ax::Object
 
     bool isDefaultRenderTarget() const { return _defaultRenderTarget; }
 
-    void addFlag(TargetBufferFlags flag) {
-        setTargetFlags(_flags |= flag);
-    }
-    void removeFlag(TargetBufferFlags flag) { 
-        setTargetFlags(_flags & ~flag);
-    }
-
-    TargetBufferFlags getTargetFlags() const { return _flags; }
-    void setTargetFlags(TargetBufferFlags flags) { 
-        _flags = flags; 
-    }
-
     void setColorAttachment(ColorAttachment attachment)
     {
         for (auto colorItem : _color)
@@ -83,12 +71,10 @@ class RenderTarget : public ax::Object
     ColorAttachment _color{};
     RenderBuffer _depth{};
     RenderBuffer _stencil{};
-    TargetBufferFlags _flags{};
 
 protected:
     bool _defaultRenderTarget = false;
     mutable bool _dirty = false;
-    // uint8_t samples = 1;
 };
 
 NS_AX_BACKEND_END

core/renderer/backend/Types.h

@@ -45,7 +45,6 @@ inline TargetBufferFlags getMRTColorFlag(size_t index) noexcept
 }
 
 typedef TargetBufferFlags ClearFlag;
-typedef TargetBufferFlags RenderTargetFlag;
 
 struct SamplerDescriptor
 {

core/renderer/backend/metal/CommandBufferMTL.h

@@ -255,7 +255,6 @@ class CommandBufferMTL : public CommandBuffer
     dispatch_semaphore_t _frameBoundarySemaphore;
     const RenderTarget* _currentRenderTarget = nil;  // weak ref
     RenderPassDescriptor _currentRenderPassDesc;
-    TargetBufferFlags _currentRenderTargetFlags = TargetBufferFlags::NONE;
     NSAutoreleasePool* _autoReleasePool         = nil;
 
     std::vector<std::pair<TextureBackend*, std::function<void(const PixelBufferDescriptor&)>>> _captureCallbacks;

core/renderer/backend/metal/CommandBufferMTL.mm

@@ -204,15 +204,13 @@ inline int clamp(int value, int min, int max)
                                                   const RenderPassDescriptor& renderPassDesc)
 {
     if (_mtlRenderEncoder != nil && _currentRenderPassDesc == renderPassDesc && _currentRenderTarget == renderTarget &&
-        _currentRenderTargetFlags == renderTarget->getTargetFlags() &&
         !renderTarget->isDirty())
     {
         return;
     }
 
     _currentRenderTarget      = renderTarget;
     _currentRenderPassDesc    = renderPassDesc;
-    _currentRenderTargetFlags = renderTarget->getTargetFlags();
 
     if (_mtlRenderEncoder != nil)
     {

core/renderer/backend/metal/DepthStencilStateMTL.h

@@ -1,5 +1,6 @@
 /****************************************************************************
  Copyright (c) 2018-2019 Xiamen Yaji Software Co., Ltd.
+ Copyright (c) 2019-present Axmol Engine contributors (see AUTHORS.md).
 
  https://axmol.dev/
 
@@ -62,6 +63,8 @@ class DepthStencilStateMTL : public DepthStencilState
 
     // the current depth stencil state
     id<MTLDepthStencilState> _mtlDepthStencilState = nil;
+    // static state for disabled depth and stencil
+    id<MTLDepthStencilState> _mtlDepthStencilDisabledState = nil;
 
     tsl::robin_map<uint32_t, id<MTLDepthStencilState>> _mtlStateCache;
 };

core/renderer/backend/metal/DepthStencilStateMTL.mm

@@ -1,5 +1,6 @@
 /****************************************************************************
  Copyright (c) 2018-2019 Xiamen Yaji Software Co., Ltd.
+ Copyright (c) 2019-present Axmol Engine contributors (see AUTHORS.md).
 
  https://axmol.dev/
 
@@ -107,15 +108,21 @@ void setMTLStencilDescriptor(MTLStencilDescriptor* stencilDescriptor, const Sten
 }
 }
 
-DepthStencilStateMTL::DepthStencilStateMTL(id<MTLDevice> mtlDevice) : _mtlDevice(mtlDevice) {}
+DepthStencilStateMTL::DepthStencilStateMTL(id<MTLDevice> mtlDevice) : _mtlDevice(mtlDevice)
+{
+    // By default MTLDepthStencilDescriptor disables depth and stencil access
+    MTLDepthStencilDescriptor* mtlDescriptor = [MTLDepthStencilDescriptor new];
+    _mtlDepthStencilDisabledState = [mtlDevice newDepthStencilStateWithDescriptor:mtlDescriptor];
+    [mtlDescriptor release];
+}
 
 void DepthStencilStateMTL::update(const DepthStencilDescriptor& dsDesc)
 {
     DepthStencilState::update(dsDesc);
 
     if (!isEnabled())
     {
-        _mtlDepthStencilState = nil;
+        _mtlDepthStencilState = _mtlDepthStencilDisabledState;
         return;
     }
 
@@ -159,6 +166,7 @@ void setMTLStencilDescriptor(MTLStencilDescriptor* stencilDescriptor, const Sten
 DepthStencilStateMTL::~DepthStencilStateMTL()
 {
     _mtlDepthStencilState = nullptr;
+    [_mtlDepthStencilDisabledState release];
     for (auto& stateItem : _mtlStateCache)
         [stateItem.second release];
     _mtlStateCache.clear();

core/renderer/backend/metal/DriverMTL.h

@@ -149,9 +149,8 @@ class DriverMTL : public DriverBase
      */
     TextureBackend* newTexture(const TextureDescriptor& descriptor) override;
 
-    RenderTarget* newDefaultRenderTarget(TargetBufferFlags rtf) override;
-    RenderTarget* newRenderTarget(TargetBufferFlags rtf,
-                                  TextureBackend* colorAttachment,
+    RenderTarget* newDefaultRenderTarget() override;
+    RenderTarget* newRenderTarget(TextureBackend* colorAttachment,
                                   TextureBackend* depthAttachment,
                                   TextureBackend* stencilAttachhment) override;
 

core/renderer/backend/metal/DriverMTL.mm

@@ -482,20 +482,17 @@ bool supportS3TC(FeatureSet featureSet)
     }
 }
 
-RenderTarget* DriverMTL::newDefaultRenderTarget(TargetBufferFlags rtf)
+RenderTarget* DriverMTL::newDefaultRenderTarget()
 {
     auto rtGL = new RenderTargetMTL(true);
-    rtGL->setTargetFlags(rtf);
     return rtGL;
 }
 
-RenderTarget* DriverMTL::newRenderTarget(TargetBufferFlags rtf,
-                                         TextureBackend* colorAttachment,
+RenderTarget* DriverMTL::newRenderTarget(TextureBackend* colorAttachment,
                                          TextureBackend* depthAttachment,
                                          TextureBackend* stencilAttachhment)
 {
     auto rtMTL = new RenderTargetMTL(false);
-    rtMTL->setTargetFlags(rtf);
     RenderTarget::ColorAttachment colors{{colorAttachment, 0}};
     rtMTL->setColorAttachment(colors);
     rtMTL->setDepthAttachment(depthAttachment);

core/renderer/backend/metal/RenderPipelineMTL.mm

@@ -308,22 +308,11 @@ MTLBlendOperation toMTLBlendOperation(BlendOperation operation)
 {
     // Choose color attachment format
     auto rtMTL   = static_cast<const RenderTargetMTL*>(renderTarget);
-    auto rtflags = rtMTL->getTargetFlags();
     for (auto i = 0; i < MAX_COLOR_ATTCHMENT; ++i)
-    {
-        colorAttachmentsFormat[i] =
-            bitmask::any(rtflags, getMRTColorFlag(i)) ? rtMTL->getColorAttachmentPixelFormat(i) : PixelFormat::NONE;
-    }
+        colorAttachmentsFormat[i] = rtMTL->getColorAttachmentPixelFormat(i);
 
-    if (bitmask::any(rtflags, RenderTargetFlag::DEPTH_AND_STENCIL))
-    {
-        depthFormat   = rtMTL->getDepthAttachmentPixelFormat();
-        stencilFormat = rtMTL->getStencilAttachmentPixelFormat();
-    }
-    else
-    {
-        depthFormat = stencilFormat = PixelFormat::NONE;
-    }
+    depthFormat   = rtMTL->getDepthAttachmentPixelFormat();
+    stencilFormat = rtMTL->getStencilAttachmentPixelFormat();
 }
 
 void RenderPipelineMTL::setBlendStateAndFormat(const BlendDescriptor& blendDescriptor)