修改yuv to rgb转换矩阵获取；

RealChuan · Nov 15, 2023 · 67e7cfe · 67e7cfe
1 parent 5078e42
commit 67e7cfe
Show file tree

Hide file tree

Showing 18 changed files with 219 additions and 141 deletions.
diff --git a/README.md b/README.md
@@ -21,59 +21,47 @@
 
 1. 参考[MPV video_shaders](https://github.com/mpv-player/mpv/blob/master/video/out/gpu/video_shaders.c#L341)，效果也不是很好；应该是哪里有遗漏。
 
-```cpp
-void pass_linearize(struct gl_shader_cache *sc, enum mp_csp_trc trc);
-void pass_delinearize(struct gl_shader_cache *sc, enum mp_csp_trc trc);
-```
+2. MPV的shader生成方式：
 
-2. 在NV12的shader上，参考MPV实现的对SMPTE2084进行图像调整shader
+    1. 根据`AVColorTransferCharacteristic`进行gamma、PQ或者HLG等等的调整，OETF；
 
-```glsl
-#version 330 core
+        ```cpp
+        void pass_linearize(struct gl_shader_cache *sc, enum mp_csp_trc trc);
+        ```
 
-in vec2 TexCord;    // 纹理坐标
-out vec4 FragColor; // 输出颜色
+        ```glsl
+        color.rgb = clamp(color.rgb, 0.0, 1.0);
+        color.rgb = pow(color.rgb, vec3(1.0 / PQ_M2));
+        color.rgb = max(color.rgb - vec3(PQ_C1), vec3(0.0)) / (vec3(PQ_C2) - vec3(PQ_C3) * color.rgb);
+        color.rgb = pow(color.rgb, vec3(1.0 / PQ_M1));
+        ```
 
-uniform sampler2D tex_y;
-uniform sampler2D tex_uv;
+    2. 色调映射，tone mapping；
 
-uniform vec3 offset;
-uniform mat3 colorConversion;
+        ```cpp
+        static void pass_tone_map(struct gl_shader_cache *sc,
+                          float src_peak, float dst_peak,
+                          const struct gl_tone_map_opts*opts);
+        ```
 
-const float PQ_M1 = 2610. / 4096 * 1. / 4, PQ_M2 = 2523. / 4096 * 128, PQ_C1 = 3424. / 4096,
-            PQ_C2 = 2413. / 4096 * 32, PQ_C3 = 2392. / 4096 * 32;
+    3. 根据`AVColorTransferCharacteristic`进行gamma、PQ或者HLG等等的调整，EOTF;
 
-#define MP_REF_WHITE 203.0
-#define MP_REF_WHITE_HLG 3.17955
+        ```cpp
+        void pass_delinearize(struct gl_shader_cache *sc, enum mp_csp_trc trc);
+        ```
 
-void main()
-{
-    vec3 yuv;
-    vec3 rgb;
-
-    yuv.x = texture(tex_y, TexCord).r;
-    yuv.yz = texture(tex_uv, TexCord).rg;
-
-    yuv += offset;
-    rgb = yuv * colorConversion;
-
-    vec4 color = vec4(rgb, 1.0);
-    // ------------------
-    color.rgb = clamp(color.rgb, 0.0, 1.0);
-    color.rgb = pow(color.rgb, vec3(1.0 / PQ_M2));
-    color.rgb = max(color.rgb - vec3(PQ_C1), vec3(0.0)) / (vec3(PQ_C2) - vec3(PQ_C3) * color.rgb);
-    color.rgb = pow(color.rgb, vec3(1.0 / PQ_M1));
-    // ------------------
-    color.rgb = clamp(color.rgb, 0.0, 1.0);
-    color.rgb = pow(color.rgb, vec3(PQ_M1));
-    color.rgb = (vec3(PQ_C1) + vec3(PQ_C2) * color.rgb) / (vec3(1.0) + vec3(PQ_C3) * color.rgb);
-    color.rgb = pow(color.rgb, vec3(PQ_M2));
-    // ------------------
-    rgb = color.rgb;
-
-    FragColor = vec4(rgb, 1.0);
-}
-```
+        ```glsl
+        color.rgb = clamp(color.rgb, 0.0, 1.0);
+        color.rgb = pow(color.rgb, vec3(PQ_M1));
+        color.rgb = (vec3(PQ_C1) + vec3(PQ_C2) * color.rgb) / (vec3(1.0) + vec3(PQ_C3) * color.rgb);
+        color.rgb = pow(color.rgb, vec3(PQ_M2));
+        ```
+
+3. MPV生成得到yuv -> RGB转换矩阵的系数；
+
+    ```cpp
+    void mp_get_csp_matrix(struct mp_csp_params *params, struct mp_cmat *m);
+    ```
 
 ### OpenGL 渲染图像，怎么实现画质增强的效果？
 

diff --git a/ffmpeg/CMakeLists.txt b/ffmpeg/CMakeLists.txt
@@ -56,6 +56,7 @@ set(PROJECT_SOURCES
     clock.hpp
     codeccontext.cpp
     codeccontext.h
+    colorspace.cc
     colorspace.hpp
     decoder.cc
     decoder.h

diff --git a/ffmpeg/colorspace.cc b/ffmpeg/colorspace.cc
@@ -0,0 +1,117 @@
+#include "colorspace.hpp"
+#include "frame.hpp"
+
+extern "C" {
+#include <libavutil/frame.h>
+#include <libavutil/pixdesc.h>
+}
+
+namespace Ffmpeg::ColorSpace {
+
+static constexpr QVector3D kJPEG_full_offset = {0.000000F, -0.501961F, -0.501961F};
+static constexpr QVector3D kBT601_limited_offset = {-0.062745F, -0.501961F, -0.501961F};
+static constexpr QVector3D kBT709_full_offset = {0.000000F, -0.501961F, -0.501961F};
+static constexpr QVector3D kBT709_limited_offset = {-0.062745F, -0.501961F, -0.501961F};
+static constexpr QVector3D kBT2020_8bit_full_offset = {0.000000F, -0.501961F, -0.501961F};
+static constexpr QVector3D kBT2020_8bit_limited_offset = {-0.062745F, -0.501961F, -0.501961F};
+static constexpr QVector3D kBT2020_10bit_full_offset = {0.000000F, -0.500489F, -0.500489F};
+static constexpr QVector3D kBT2020_10bit_limited_offset = {-0.062561F, -0.500489F, -0.500489F};
+static constexpr QVector3D kBT2020_12bit_full_offset = {0.000000F, -0.500122F, -0.500122F};
+static constexpr QVector3D kBT2020_12bit_limited_offset = {-0.062515F, -0.500122F, -0.500122F};
+
+static constexpr float kJPEG_full_yuv_to_rgb[3][3] = {{1.000000F, 1.000000F, 1.000000F},
+                                                      {-0.000000F, -0.344136F, 1.772000F},
+                                                      {1.402000F, -0.714136F, 0.000000F}};
+static constexpr float kRec601_limited_yuv_to_rgb[3][3] = {{1.164384F, 1.164384F, 1.164384F},
+                                                           {-0.000000F, -0.391762F, 2.017232F},
+                                                           {1.596027F, -0.812968F, 0.000000F}};
+static constexpr float kRec709_full_yuv_to_rgb[3][3] = {{1.000000F, 1.000000F, 1.000000F},
+                                                        {-0.000000F, -0.187324F, 1.855600F},
+                                                        {1.574800F, -0.468124F, -0.000000F}};
+static constexpr float kRec709_limited_yuv_to_rgb[3][3] = {{1.164384F, 1.164384F, 1.164384F},
+                                                           {-0.000000F, -0.213249F, 2.112402F},
+                                                           {1.792741F, -0.532909F, -0.000000F}};
+static constexpr float kBT2020_8bit_full_yuv_to_rgb[3][3] = {{1.000000F, 1.000000F, 1.000000F},
+                                                             {-0.000000F, -0.164553F, 1.881400F},
+                                                             {1.474600F, -0.571353F, -0.000000F}};
+static constexpr float kBT2020_8bit_limited_yuv_to_rgb[3][3] = {{1.164384F, 1.164384F, 1.164384F},
+                                                                {-0.000000F, -0.187326F, 2.141772F},
+                                                                {1.678674F, -0.650424F, -0.000000F}};
+static constexpr float kBT2020_10bit_full_yuv_to_rgb[3][3] = {{1.000000F, 1.000000F, 1.000000F},
+                                                              {-0.000000F, -0.164553F, 1.881400F},
+                                                              {1.474600F, -0.571353F, -0.000000F}};
+static constexpr float kBT2020_10bit_limited_yuv_to_rgb[3][3]
+    = {{1.167808F, 1.167808F, 1.167808F},
+       {-0.000000F, -0.187877F, 2.148072F},
+       {1.683611F, -0.652337F, -0.000000F}};
+static constexpr float kBT2020_12bit_full_yuv_to_rgb[3][3] = {{1.000000F, 1.000000F, 1.000000F},
+                                                              {-0.000000F, -0.164553F, 1.881400F},
+                                                              {1.474600F, -0.571353F, -0.000000F}};
+static constexpr float kBT2020_12bit_limited_yuv_to_rgb[3][3]
+    = {{1.168664F, 1.168664F, 1.168664F},
+       {-0.000000F, -0.188015F, 2.149647F},
+       {1.684846F, -0.652816F, -0.000000F}};
+
+auto getYuvToRgbParam(Frame *frame) -> YuvToRgbParam
+{
+    auto *avFrame = frame->avFrame();
+    bool isFullRange = avFrame->color_range == AVCOL_RANGE_JPEG;
+    YuvToRgbParam param;
+    switch (avFrame->colorspace) {
+    case AVCOL_SPC_BT709:
+        if (isFullRange) {
+            param.offset = kBT709_full_offset;
+            param.matrix = QMatrix3x3(&kRec709_full_yuv_to_rgb[0][0]);
+        } else {
+            param.offset = kBT709_limited_offset;
+            param.matrix = QMatrix3x3(&kRec709_limited_yuv_to_rgb[0][0]);
+        }
+        break;
+    case AVCOL_SPC_BT2020_NCL: {
+        int bitsPerPixel = av_get_bits_per_pixel(
+            av_pix_fmt_desc_get(static_cast<AVPixelFormat>(avFrame->format)));
+        switch (bitsPerPixel) {
+        case 8:
+            if (isFullRange) {
+                param.offset = kBT2020_8bit_full_offset;
+                param.matrix = QMatrix3x3(&kBT2020_8bit_full_yuv_to_rgb[0][0]);
+            } else {
+                param.offset = kBT2020_8bit_limited_offset;
+                param.matrix = QMatrix3x3(&kBT2020_8bit_limited_yuv_to_rgb[0][0]);
+            }
+            break;
+        case 12:
+            if (isFullRange) {
+                param.offset = kBT2020_12bit_full_offset;
+                param.matrix = QMatrix3x3(&kBT2020_12bit_full_yuv_to_rgb[0][0]);
+            } else {
+                param.offset = kBT2020_12bit_limited_offset;
+                param.matrix = QMatrix3x3(&kBT2020_12bit_limited_yuv_to_rgb[0][0]);
+            }
+            break;
+        default:
+            if (isFullRange) {
+                param.offset = kBT2020_10bit_full_offset;
+                param.matrix = QMatrix3x3(&kBT2020_10bit_full_yuv_to_rgb[0][0]);
+            } else {
+                param.offset = kBT2020_10bit_limited_offset;
+                param.matrix = QMatrix3x3(&kBT2020_10bit_limited_yuv_to_rgb[0][0]);
+            }
+            break;
+        }
+        break;
+    }
+    default:
+        if (isFullRange) {
+            param.offset = kJPEG_full_offset;
+            param.matrix = QMatrix3x3(&kJPEG_full_yuv_to_rgb[0][0]);
+        } else {
+            param.offset = kBT601_limited_offset;
+            param.matrix = QMatrix3x3(&kRec601_limited_yuv_to_rgb[0][0]);
+        }
+        break;
+    };
+    return param;
+}
+
+} // namespace Ffmpeg::ColorSpace
diff --git a/ffmpeg/colorspace.hpp b/ffmpeg/colorspace.hpp
@@ -1,32 +1,25 @@
 #ifndef COLORSPACE_HPP
 #define COLORSPACE_HPP
 
+#include <QGenericMatrix>
 #include <QVector3D>
 
-#include <array>
+namespace Ffmpeg {
 
-namespace Ffmpeg::ColorSpace {
+class Frame;
 
-static constexpr QVector3D kJPEGOffset = {0, -0.501960814, -0.501960814};
+namespace ColorSpace {
 
-static constexpr std::array<float, 9> kJPEGMatrix
-    = {1, 1, 1, 0.000, -0.3441, 1.772, 1.402, -0.7141, 0.000};
+struct YuvToRgbParam
+{
+    QVector3D offset;
+    QMatrix3x3 matrix;
+};
 
-static constexpr QVector3D kBT601Offset = {-0.0627451017, -0.501960814, -0.501960814};
+auto getYuvToRgbParam(Frame *frame) -> YuvToRgbParam;
 
-static constexpr std::array<float, 9> kBT601Matrix
-    = {1.1644, 1.1644, 1.1644, 0.000, -0.3918, 2.0172, 1.596, -0.813, 0.000};
+} // namespace ColorSpace
 
-static constexpr QVector3D kBT7090ffset = {-0.0627451017, -0.501960814, -0.501960814};
-
-static constexpr std::array<float, 9> kBT709Matrix
-    = {1.1644, 1.1644, 1.1644, 0.000, -0.2132, 2.112, 1.7927, -0.5329, 0.000};
-
-static constexpr QVector3D kBT2020ffset = {-0.0627451017, -0.501960814, -0.501960814};
-
-static constexpr std::array<float, 9> kBT2020Matrix
-    = {1.1678, 1.1678, 1.1678, 0.0000, -0.1879, 2.1481, 1.6836, -0.6523, 0.0000};
-
-} // namespace Ffmpeg::ColorSpace
+} // namespace Ffmpeg
 
 #endif // COLORSPACE_HPP
diff --git a/ffmpeg/ffmpeg.pro b/ffmpeg/ffmpeg.pro
@@ -24,6 +24,7 @@ SOURCES += \
     averrormanager.cc \
     clock.cc \
     codeccontext.cpp \
+    colorspace.cc \
     decoder.cc \
     ffmpegutils.cc \
     formatcontext.cpp \

diff --git a/ffmpeg/videorender/openglrender.cc b/ffmpeg/videorender/openglrender.cc
@@ -182,32 +182,6 @@ void OpenglRender::initSubTexture()
     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 }
 
-void OpenglRender::setColorSpace()
-{
-    auto *avFrame = d_ptr->framePtr->avFrame();
-    switch (avFrame->colorspace) {
-    case AVCOL_SPC_BT470BG:
-    case AVCOL_SPC_SMPTE170M:
-        d_ptr->programPtr->setUniformValue("offset", ColorSpace::kBT601Offset);
-        d_ptr->programPtr->setUniformValue("colorConversion",
-                                           QMatrix3x3(ColorSpace::kBT601Matrix.data()));
-        break;
-    case AVCOL_SPC_BT2020_NCL:
-    case AVCOL_SPC_BT2020_CL:
-        d_ptr->programPtr->setUniformValue("offset", ColorSpace::kBT2020ffset);
-        d_ptr->programPtr->setUniformValue("colorConversion",
-                                           QMatrix3x3(ColorSpace::kBT2020Matrix.data()));
-
-        break;
-    //case AVCOL_SPC_BT709:
-    default:
-        d_ptr->programPtr->setUniformValue("offset", ColorSpace::kBT7090ffset);
-        d_ptr->programPtr->setUniformValue("colorConversion",
-                                           QMatrix3x3(ColorSpace::kBT709Matrix.data()));
-        break;
-    }
-}
-
 void OpenglRender::setColorTrc()
 {
     auto *avFrame = d_ptr->framePtr->avFrame();
@@ -318,7 +292,9 @@ void OpenglRender::paintVideoFrame()
     d_ptr->programPtr->setUniformValue("contrast", m_colorSpaceTrc.contrast);
     d_ptr->programPtr->setUniformValue("saturation", m_colorSpaceTrc.saturation);
     d_ptr->programPtr->setUniformValue("brightness", m_colorSpaceTrc.brightness);
-    setColorSpace();
+    auto param = Ffmpeg::ColorSpace::getYuvToRgbParam(d_ptr->framePtr.data());
+    d_ptr->programPtr->setUniformValue("offset", param.offset);
+    d_ptr->programPtr->setUniformValue("colorConversion", param.matrix);
     setColorTrc();
     draw();
     d_ptr->programPtr->release();

diff --git a/ffmpeg/videorender/openglrender.hpp b/ffmpeg/videorender/openglrender.hpp
@@ -48,7 +48,6 @@ class FFMPEG_EXPORT OpenglRender : public VideoRender,
 
     void paintVideoFrame();
     void paintSubTitleFrame();
-    void setColorSpace();
     void setColorTrc();
 
     void updateYUV420P();

diff --git a/ffmpeg/videorender/openglshader.hpp b/ffmpeg/videorender/openglshader.hpp
@@ -11,7 +11,7 @@ class OpenglShader : public QObject
     explicit OpenglShader(QObject *parent = nullptr);
     ~OpenglShader() override;
 
-    QByteArray generate(int format);
+    auto generate(int format) -> QByteArray;
 
 private:
     class OpenglShaderPrivate;

diff --git a/ffmpeg/videorender/shader/video_bgr24.frag b/ffmpeg/videorender/shader/video_bgr24.frag
@@ -1,10 +1,11 @@
 void main()
 {
-    vec3 rgb = texture(tex_y, TexCord).bgr;
+    vec4 color = vec4(0.0, 0.0, 0.0, 1.0);
+    color.rgb = texture(tex_y, TexCord).bgr;
 
-    rgb = adjustContrast(rgb, contrast);
-    rgb = adjustSaturation(rgb, saturation);
-    rgb = adjustBrightness(rgb, brightness);
+    color.rgb = adjustContrast(color.rgb, contrast);
+    color.rgb = adjustSaturation(color.rgb, saturation);
+    color.rgb = adjustBrightness(color.rgb, brightness);
 
-    FragColor = vec4(rgb, 1.0);
+    FragColor = color;
 }
diff --git a/ffmpeg/videorender/shader/video_color.frag b/ffmpeg/videorender/shader/video_color.frag
@@ -8,7 +8,8 @@ vec3 adjustBrightness(vec3 rgb, float brightness) // 调整亮度
     return clamp(rgb + brightness, 0.0, 1.0);
 }
 
-vec3 adjustContrast(vec3 rgb, float contrast) // 调整对比度 和使用sws_setColorspaceDetails效果不太一样
+// 调整对比度 和使用sws_setColorspaceDetails效果不太一样
+vec3 adjustContrast(vec3 rgb, float contrast)
 {
     contrast = clamp(contrast, 0.0, 2.0);
     vec3 contrastColor = (rgb - vec3(0.5)) * contrast + vec3(0.5);

diff --git a/ffmpeg/videorender/shader/video_nv12.frag b/ffmpeg/videorender/shader/video_nv12.frag
@@ -1,17 +1,17 @@
 void main()
 {
     vec3 yuv;
-    vec3 rgb;
+    vec4 color = vec4(0.0, 0.0, 0.0, 1.0);
 
     yuv.x = texture(tex_y, TexCord).r;
     yuv.yz = texture(tex_u, TexCord).rg;
 
     yuv += offset;
-    rgb = yuv * colorConversion;
+    color.rgb = yuv * colorConversion;
 
-    rgb = adjustContrast(rgb, contrast);
-    rgb = adjustSaturation(rgb, saturation);
-    rgb = adjustBrightness(rgb, brightness);
+    color.rgb = adjustContrast(color.rgb, contrast);
+    color.rgb = adjustSaturation(color.rgb, saturation);
+    color.rgb = adjustBrightness(color.rgb, brightness);
 
-    FragColor = vec4(rgb, 1.0);
+    FragColor = color;
 }