diff --git a/app/src/streaming/video/deko3d/DKVideoRenderer.cpp b/app/src/streaming/video/deko3d/DKVideoRenderer.cpp index 87effa9..7490169 100644 --- a/app/src/streaming/video/deko3d/DKVideoRenderer.cpp +++ b/app/src/streaming/video/deko3d/DKVideoRenderer.cpp @@ -52,6 +52,13 @@ namespace { static constexpr unsigned StaticCmdSize = 0x10000; + struct Transformation + { + glm::mat3 yuvmat; + glm::vec3 offset; + glm::vec4 uv_data; + }; + struct Vertex { float position[3]; @@ -98,7 +105,7 @@ void DKVideoRenderer::checkAndInitialize(int width, int height, AVFrame* frame) m_screen_width = width; m_screen_height = height; - auto *vctx = (brls::SwitchVideoContext *)brls::Application::getPlatform()->getVideoContext(); + vctx = (brls::SwitchVideoContext *)brls::Application::getPlatform()->getVideoContext(); this->dev = vctx->getDeko3dDevice(); this->queue = vctx->getQueue(); @@ -123,6 +130,35 @@ void DKVideoRenderer::checkAndInitialize(int width, int height, AVFrame* frame) vertexBuffer = pool_data->allocate(sizeof(QuadVertexData), alignof(Vertex)); memcpy(vertexBuffer.getCpuAddr(), QuadVertexData.data(), vertexBuffer.getSize()); + +// Load the transform buffer + transformUniformBuffer = pool_data->allocate(sizeof(Transformation), DK_UNIFORM_BUF_ALIGNMENT); + auto transformState = reinterpret_cast(transformUniformBuffer.getCpuAddr()); + + bool colorFull = frame->color_range == AVCOL_RANGE_JPEG; + + transformState->offset = gl_color_offset(colorFull); + transformState->yuvmat = gl_color_matrix(frame->colorspace, colorFull); + + float frameAspect = ((float)m_frame_height / (float)m_frame_width); + float screenAspect = ((float)m_screen_height / (float)m_screen_width); + + if (frameAspect > screenAspect) { + float multiplier = frameAspect / screenAspect; + transformState->uv_data = { 0.5f - 0.5f * (1.0f / multiplier), + 0.0f, multiplier, 1.0f }; + } else { + float multiplier = screenAspect / frameAspect; + transformState->uv_data = { 0.0f, + 0.5f - 0.5f * (1.0f / multiplier), 1.0f, multiplier }; + } + + + // cmdbuf.pushConstants( + // transformUniformBuffer.getGpuAddr(), transformUniformBuffer.getSize(), + // 0, sizeof(transformState), &transformState); + + // Allocate image indexes for planes lumaTextureId = vctx->allocateImageIndex(); chromaTextureId = vctx->allocateImageIndex(); @@ -161,48 +197,10 @@ void DKVideoRenderer::checkAndInitialize(int width, int height, AVFrame* frame) imageDescriptorSet->update(cmdbuf, lumaTextureId, lumaDesc); imageDescriptorSet->update(cmdbuf, chromaTextureId, chromaDesc); -// Load the transform buffer - transformUniformBuffer = pool_data->allocate(sizeof(transformState), DK_UNIFORM_BUF_ALIGNMENT); - - bool colorFull = frame->color_range == AVCOL_RANGE_JPEG; - - transformState.offset = gl_color_offset(colorFull); - transformState.yuvmat = gl_color_matrix(frame->colorspace, colorFull); - - float frameAspect = ((float)m_frame_height / (float)m_frame_width); - float screenAspect = ((float)m_screen_height / (float)m_screen_width); - - if (frameAspect > screenAspect) { - float multiplier = frameAspect / screenAspect; - transformState.uv_data = { 0.5f - 0.5f * (1.0f / multiplier), - 0.0f, multiplier, 1.0f }; - } else { - float multiplier = screenAspect / frameAspect; - transformState.uv_data = { 0.0f, - 0.5f - 0.5f * (1.0f / multiplier), 1.0f, multiplier }; - } - - cmdbuf.pushConstants( - transformUniformBuffer.getGpuAddr(), transformUniformBuffer.getSize(), - 0, sizeof(transformState), &transformState); - queue.submitCommands(cmdbuf.finishList()); queue.waitIdle(); - m_is_initialized = true; -} - -int frames = 0; -uint64_t timeCount = 0; - -void DKVideoRenderer::draw(NVGcontext* vg, int width, int height, AVFrame* frame, int imageFormat) { - checkAndInitialize(width, height, frame); - uint64_t before_render = LiGetMillis(); - - if (!m_video_render_stats.rendered_frames) { - m_video_render_stats.measurement_start_timestamp = before_render; - } dk::RasterizerState rasterizerState; dk::ColorState colorState; @@ -212,13 +210,11 @@ void DKVideoRenderer::draw(NVGcontext* vg, int width, int height, AVFrame* frame cmdbuf.clear(); cmdbuf.clearColor(0, DkColorMask_RGBA, 0.0f, 0.0f, 0.0f, 0.0f); - // brls::Logger::debug("TIME LOG 1: {}", float(LiGetMillis() - before_render)); - // Bind state required for drawing the triangle cmdbuf.bindShaders(DkStageFlag_GraphicsMask, { vertexShader, fragmentShader }); cmdbuf.bindTextures(DkStage_Fragment, 0, dkMakeTextureHandle(lumaTextureId, 0)); cmdbuf.bindTextures(DkStage_Fragment, 1, dkMakeTextureHandle(chromaTextureId, 0)); - cmdbuf.bindUniformBuffer(DkStage_Vertex, 0, transformUniformBuffer.getGpuAddr(), transformUniformBuffer.getSize()); + cmdbuf.bindUniformBuffer(DkStage_Fragment, 2, transformUniformBuffer.getGpuAddr(), transformUniformBuffer.getSize()); cmdbuf.bindRasterizerState(rasterizerState); cmdbuf.bindColorState(colorState); cmdbuf.bindColorWriteState(colorWriteState); @@ -226,19 +222,30 @@ void DKVideoRenderer::draw(NVGcontext* vg, int width, int height, AVFrame* frame cmdbuf.bindVtxAttribState(VertexAttribState); cmdbuf.bindVtxBufferState(VertexBufferState); - // brls::Logger::debug("TIME LOG 2: {}", float(LiGetMillis() - before_render)); - // Draw the triangle cmdbuf.draw(DkPrimitive_Quads, QuadVertexData.size(), 1, 0, 0); + cmdlist = cmdbuf.finishList(); - // brls::Logger::debug("TIME LOG 3: {}", float(LiGetMillis() - before_render)); + m_is_initialized = true; +} + +int frames = 0; +uint64_t timeCount = 0; + +void DKVideoRenderer::draw(NVGcontext* vg, int width, int height, AVFrame* frame, int imageFormat) { + checkAndInitialize(width, height, frame); + + uint64_t before_render = LiGetMillis(); + + if (!m_video_render_stats.rendered_frames) { + m_video_render_stats.measurement_start_timestamp = before_render; + } // Finish off this command list - queue.submitCommands(cmdbuf.finishList()); + queue = vctx->getQueue(); + queue.submitCommands(cmdlist); queue.waitIdle(); - // brls::Logger::debug("TIME LOG 4: {}", float(LiGetMillis() - before_render)); - frames++; timeCount += LiGetMillis() - before_render; diff --git a/app/src/streaming/video/deko3d/DKVideoRenderer.hpp b/app/src/streaming/video/deko3d/DKVideoRenderer.hpp index 3f8ba37..3daa0ac 100644 --- a/app/src/streaming/video/deko3d/DKVideoRenderer.hpp +++ b/app/src/streaming/video/deko3d/DKVideoRenderer.hpp @@ -7,18 +7,12 @@ #include #include +#include #include #include #include #include -struct Transformation -{ - glm::mat3 yuvmat; - glm::vec3 offset; - glm::vec4 uv_data; -}; - class DKVideoRenderer : public IVideoRenderer { public: DKVideoRenderer(); @@ -38,9 +32,7 @@ class DKVideoRenderer : public IVideoRenderer { int m_screen_width = 0; int m_screen_height = 0; - Transformation transformState; - CMemPool::Handle transformUniformBuffer; - + brls::SwitchVideoContext *vctx = nullptr; dk::Device dev; dk::Queue queue; @@ -49,6 +41,7 @@ class DKVideoRenderer : public IVideoRenderer { std::optional pool_data; dk::UniqueCmdBuf cmdbuf; + DkCmdList cmdlist; CDescriptorSet<4096U> *imageDescriptorSet; // CDescriptorSet<1> samplerDescriptorSet; @@ -57,6 +50,7 @@ class DKVideoRenderer : public IVideoRenderer { CShader fragmentShader; CMemPool::Handle vertexBuffer; + CMemPool::Handle transformUniformBuffer; dk::ImageLayout lumaMappingLayout; dk::ImageLayout chromaMappingLayout; diff --git a/app/src/streaming/video/deko3d/texture_fsh.glsl b/app/src/streaming/video/deko3d/texture_fsh.glsl index 0a47d7e..4598f5e 100644 --- a/app/src/streaming/video/deko3d/texture_fsh.glsl +++ b/app/src/streaming/video/deko3d/texture_fsh.glsl @@ -15,7 +15,29 @@ layout (std140, binding = 2) uniform Transformation void main() { - vec2 uv = (vTextureCoord - u.uv_data.xy) * u.uv_data.zw; - vec3 YCbCr = vec3(texture(plane0, uv).r, texture(plane1, uv).r, texture(plane1, uv).g) - u.offset; - outColor = vec4(clamp(u.yuvmat * YCbCr, 0.0, 1.0), 1.0); + // Not work + // vec2 uv = (vTextureCoord - u.uv_data.xy) * u.uv_data.zw; + // vec3 YCbCr = vec3(texture(plane0, uv).r, texture(plane1, uv).r, texture(plane1, uv).g) - u.offset; + // outColor = vec4(clamp(u.yuvmat * YCbCr, 0.0, 1.0), 1.0); + + // Almost work + // vec3 YCbCr = vec3( + // texture2D(plane0, vTextureCoord)[0], + // texture2D(plane1, vTextureCoord).xy + // ); + + // YCbCr -= u.offset; + // outColor = vec4(clamp(u.yuvmat * YCbCr, 0.0, 1.0), 1.0f); + + float r, g, b, yt, ut, vt; + + yt = texture2D(plane0, vTextureCoord).r; + ut = texture2D(plane1, vTextureCoord).r - 0.5; + vt = texture2D(plane1, vTextureCoord).g - 0.5; + + r = yt + 1.13983*vt; + g = yt - 0.39465*ut - 0.58060*vt; + b = yt + 2.03211*ut; + + outColor = vec4(r, g, b, 1.0); }