Add render I420 video support, fix axmolengine#2049 (axmolengine#2050)

core/media/AndroidMediaEngine.cpp

@@ -44,15 +44,16 @@ JNIEXPORT void JNICALL Java_org_axmol_lib_AxmolMediaEngine_nativeHandleVideoSamp
                                                                               int outputY,
                                                                               int videoX,
                                                                               int videoY,
-                                                                              int rotation)
+                                                                              int rotation,
+                                                                              int videoPF)
 {
     auto mediaEngine = (ax::AndroidMediaEngine*)((uintptr_t)pME);
     if (!mediaEngine)
         return;
 
     auto sampleData = static_cast<uint8_t*>(env->GetDirectBufferAddress(sampleBuffer));
 
-    mediaEngine->handleVideoSample(sampleData, sampleLen, outputX, outputY, videoX, videoY, rotation);
+    mediaEngine->handleVideoSample(sampleData, sampleLen, outputX, outputY, videoX, videoY, rotation, videoPF);
 }
 
 JNIEXPORT void JNICALL Java_org_axmol_lib_AxmolMediaEngine_nativeSetDuration(JNIEnv* env,
@@ -161,7 +162,7 @@ bool AndroidMediaEngine::transferVideoFrame()
             auto& buffer = _frameBuffer2;
 
             ax::MEVideoFrame frame{buffer.data(), buffer.data() + _outputDim.x * _outputDim.y, buffer.size(),
-                                   ax::MEVideoPixelDesc{ax::MEVideoPixelFormat::NV12, _outputDim}, _videoDim};
+                                   ax::MEVideoPixelDesc{static_cast<ax::MEVideoPixelFormat>(_videoPF), _outputDim}, _videoDim};
             frame._vpd._rotation = _videoRotation;
             assert(static_cast<int>(frame._dataLen) >= frame._vpd._dim.x * frame._vpd._dim.y * 3 / 2);
             _onVideoFrame(frame);
@@ -178,13 +179,15 @@ void AndroidMediaEngine::handleVideoSample(const uint8_t* buf,
                                            int outputY,
                                            int videoX,
                                            int videoY,
-                                           int rotation)
+                                           int rotation,
+                                           int videoPF)
 {
     std::unique_lock<std::mutex> lck(_frameBuffer1Mtx);
     _frameBuffer1.assign(buf, buf + len);
     _outputDim.set(outputX, outputY);
     _videoDim.set(videoX, videoY);
     _videoRotation = rotation;
+    _videoPF       = videoPF;
 }
 
 NS_AX_END

core/media/AndroidMediaEngine.h

@@ -59,7 +59,7 @@ class AndroidMediaEngine : public MediaEngine
     MEMediaState getState() const override;
     bool transferVideoFrame() override;
 
-    void handleVideoSample(const uint8_t* buf, size_t len, int outputX, int outputY, int videoX, int videoY, int rotation);
+    void handleVideoSample(const uint8_t* buf, size_t len, int outputX, int outputY, int videoX, int videoY, int rotation, int videoPF);
     void updateCurrentTime(double currentTime) { _currentTime = currentTime; }
     void updateDuration(double duration) { _duration = duration; }
 
@@ -71,6 +71,7 @@ class AndroidMediaEngine : public MediaEngine
     MEIntPoint _outputDim;
     MEIntPoint _videoDim;
     int _videoRotation{0};
+    int _videoPF{-1};
 
     yasio::byte_buffer _frameBuffer1;  // for write
     yasio::byte_buffer _frameBuffer2;  // for read

core/media/MediaEngine.h

@@ -98,11 +98,12 @@ enum class MEMediaState
 
 enum class MEVideoPixelFormat
 {
-    INVALID,
-    YUY2,
-    NV12,  // '420v' '420f'
+    INVALID = -1,
     RGB32,
     BGR32,
+    YUY2,
+    NV12,  // '420v' '420f'
+    I420
 };
 
 struct MEIntPoint

core/platform/android/java/src/org/axmol/lib/AxmolMediaEngine.java

@@ -26,6 +26,7 @@ of this software and associated documentation files (the "Software"), to deal
 import android.app.Activity;
 import android.content.Context;
 import android.graphics.Point;
+import android.media.MediaCodecInfo;
 import android.media.MediaFormat;
 import android.net.Uri;
 import android.os.Handler;
@@ -62,6 +63,10 @@ public class AxmolMediaEngine extends DefaultRenderersFactory implements Player.
     public static final int EVENT_STOPPED = 2;
     public static final int EVENT_ERROR = 3;
 
+    // The native video pixel formats, match with MEVideoPixelFormat
+    public static final int VIDEO_PF_NV12 = 3;
+    public static final int VIDEO_PF_I420 = 4;
+
     /** Media has been closed and cannot be played again. */
     public static final int STATE_CLOSED = 0;
 
@@ -100,11 +105,12 @@ public class AxmolMediaEngine extends DefaultRenderersFactory implements Player.
     private AtomicInteger mState = new AtomicInteger(STATE_CLOSED);
     Point mOutputDim = new Point(); // The output dim match with buffer
     Point mVideoDim = new Point(); // The video dim (validate image dim)
+    private int mVideoPF = -1;
     private int mVideoRotation = 0;
 
     /** ------ native methods ------- */
     public static native void nativeHandleEvent(long nativeObj, int arg1);
-    public static native void nativeHandleVideoSample(long nativeObj, ByteBuffer sampleData, int sampleLen, int outputX, int outputY, int videoX, int videoY, int rotation);
+    public static native void nativeHandleVideoSample(long nativeObj, ByteBuffer sampleData, int sampleLen, int outputX, int outputY, int videoX, int videoY, int rotation, int videoPF);
     public static native void nativeSetDuration(long nativeObj, double duration);
     public static native void nativeSetCurrentTime(long nativeObj, double currentTime);
 
@@ -306,10 +312,23 @@ public void onVideoFrameAboutToBeRendered(
     /** update video informations */
     private void updateVideoMeta() {
         MediaFormat format = mOutputFormat;
-        // String mimeType = format.getString(MediaFormat.KEY_MIME); // "video/raw" (NV12)
-        // Integer colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT);
-        // boolean NV12 = colorFormat == MediaCodecVideoRenderer.DESIRED_PIXEL_FORMAT;
         if(format != null) {
+            // String mimeType = format.getString(MediaFormat.KEY_MIME); // "video/raw"
+            // Note: some android 11 and older devices not response desired color format(NV12), instead will be YUV420P aka I420
+            // refer: https://github.com/axmolengine/axmol/issues/2049
+            Integer colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT);
+            switch(colorFormat) {
+                case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar:
+                    mVideoPF = VIDEO_PF_NV12;
+                    break;
+                case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar:
+                    mVideoPF = VIDEO_PF_I420;
+                    break;
+                default:
+                    mVideoPF = VIDEO_PF_NV12;
+                    Log.w(TAG, String.format("Unsupported color format: %d, video render may incorrect!", colorFormat));
+            }
+
             mOutputDim.x = format.getInteger(MediaFormat.KEY_WIDTH);
             if (format.containsKey(MediaFormat.KEY_CROP_LEFT)
                 && format.containsKey(MediaFormat.KEY_CROP_RIGHT)) {
@@ -343,7 +362,7 @@ public void processVideoFrame(MediaCodecAdapter codec, int index, long presentat
         }
 
         ByteBuffer tmpBuffer = codec.getOutputBuffer(index);
-        nativeHandleVideoSample(mNativeObj, tmpBuffer, tmpBuffer.remaining(), mOutputDim.x, mOutputDim.y, mVideoDim.x, mVideoDim.y, mVideoRotation);
+        nativeHandleVideoSample(mNativeObj, tmpBuffer, tmpBuffer.remaining(), mOutputDim.x, mOutputDim.y, mVideoDim.x, mVideoDim.y, mVideoRotation, mVideoPF);
 
         AxmolEngine.getActivity().runOnUiThread(() -> {
             if (mPlayer != null) {

core/renderer/Shaders.cpp

@@ -62,6 +62,7 @@ AX_DLL const std::string_view hsv_frag                             = "hsv_fs"sv;
 AX_DLL const std::string_view dualSampler_hsv_frag                 = "dualSampler_hsv_fs"sv;
 AX_DLL const std::string_view videoTextureYUY2_frag                = "videoTextureYUY2_fs"sv;
 AX_DLL const std::string_view videoTextureNV12_frag                = "videoTextureNV12_fs"sv;
+AX_DLL const std::string_view videoTextureI420_frag                = "videoTextureI420_fs"sv;
 AX_DLL const std::string_view lineColor_frag                       = "lineColor_fs"sv;
 AX_DLL const std::string_view lineColor_vert                       = "lineColor_vs"sv;
 AX_DLL const std::string_view color_frag                           = "color_fs"sv;

core/renderer/Shaders.h

@@ -73,6 +73,7 @@ extern AX_DLL const std::string_view dualSampler_hsv_frag;
 
 extern AX_DLL const std::string_view videoTextureYUY2_frag;
 extern AX_DLL const std::string_view videoTextureNV12_frag;
+extern AX_DLL const std::string_view videoTextureI420_frag;
 
 /* below is 3d shaders */
 extern AX_DLL const std::string_view lineColor_frag;

core/renderer/backend/Enums.h

@@ -381,6 +381,7 @@ struct ProgramType
 
         VIDEO_TEXTURE_YUY2,
         VIDEO_TEXTURE_NV12,
+        VIDEO_TEXTURE_I420, // For some android 11 and older devices
         VIDEO_TEXTURE_BGR32,
 
         BUILTIN_COUNT,

core/renderer/backend/ProgramManager.cpp

@@ -152,6 +152,8 @@ bool ProgramManager::init()
                     VertexLayoutType::Sprite);
     registerProgram(ProgramType::VIDEO_TEXTURE_NV12, positionTextureColor_vert, videoTextureNV12_frag,
                     VertexLayoutType::Sprite);
+    registerProgram(ProgramType::VIDEO_TEXTURE_I420, positionTextureColor_vert, videoTextureI420_frag,
+                    VertexLayoutType::Sprite);
 
     // The builtin dual sampler shader registry
     ProgramStateRegistry::getInstance()->registerProgram(ProgramType::POSITION_TEXTURE_COLOR,

core/renderer/shaders/videoTextureI420.frag

@@ -0,0 +1,35 @@
+#version 310 es
+precision highp float;
+precision highp int;
+
+#include "base.glsl"
+#include "colorUtils.glsl"
+
+layout(location = COLOR0) in vec4 v_color;
+layout(location = TEXCOORD0) in vec2 v_texCoord;
+
+layout(binding = 0) uniform sampler2D u_tex0; // Y sample: LumaTexture
+layout(binding = 1) uniform sampler2D u_tex1; // U sample: ChromaTexture
+layout(binding = 2) uniform sampler2D u_tex2; // V sample: ChromaTexture
+
+layout(std140) uniform fs_ub {
+    mat4 colorTransform; 
+};
+
+layout(location = SV_Target0) out vec4 FragColor;
+
+void main()
+{
+    vec3 YUV;
+
+    YUV.x = texture(u_tex0, v_texCoord).x; // Y
+    YUV.y = texture(u_tex1, v_texCoord).x; // U
+    YUV.z = texture(u_tex2, v_texCoord).x; // V
+
+    /* Convert YUV to RGB */
+    vec4 OutColor;
+    OutColor.xyz = trasnformYUV(YUV, colorTransform);
+    OutColor.w = 1.0;
+
+    FragColor = v_color * OutColor;
+}

core/ui/UIMediaPlayer.cpp

@@ -50,14 +50,14 @@ USING_NS_AX;
         do                                                          \
         {                                                           \
             decltype(value) __v = value;                            \
-            auto __loc          = (ps)->getUniformLocation(name); \
+            auto __loc          = (ps) -> getUniformLocation(name); \
             (ps)->setUniform(__loc, &__v, sizeof(__v));             \
         } while (false)
 
 #    define PS_SET_UNIFORM_R(ps, name, value)               \
         do                                                  \
         {                                                   \
-            auto __loc = (ps)->getUniformLocation(name);    \
+            auto __loc = (ps) -> getUniformLocation(name);  \
             (ps)->setUniform(__loc, &value, sizeof(value)); \
         } while (false)
 
@@ -67,10 +67,11 @@ namespace
 {
 struct PrivateVideoDescriptor
 {
-    MediaEngine* _engine       = nullptr;
-    Texture2D* _vtexture       = nullptr;
-    Texture2D* _vchromaTexture = nullptr;
-    Sprite* _vrender           = nullptr;
+    MediaEngine* _engine        = nullptr;
+    Texture2D* _vtexture        = nullptr;
+    Texture2D* _vchromaTexture  = nullptr;
+    Texture2D* _vchroma2Texture = nullptr;
+    Sprite* _vrender            = nullptr;
 
     MEVideoPixelDesc _vpixelDesc;
 
@@ -518,11 +519,13 @@ void BasicMediaController::initRenderer()
     // loop. This is a work-around for a RenderTexture issue
     // when being created such places as a button click event handler
     // on Apple platforms/Metal renderer backend
-    scheduleOnce([this](float){
+    scheduleOnce(
+        [this](float) {
         createControls();
         updateControlsForContentSize(_mediaPlayer->getContentSize());
         updateControllerState();
-    }, 0.f, "__create_video_controls"sv);
+    },
+        0.f, "__create_video_controls"sv);
 }
 
 void BasicMediaController::onPressStateChangedToPressed()
@@ -551,8 +554,10 @@ void BasicMediaController::onPressStateChangedToPressed()
                 _controlPanel->runAction(Sequence::create(FadeOut::create(0.5f), nullptr));
                 _mediaOverlay->runAction(Sequence::create(FadeOut::create(0.5f), nullptr));
             }
-        }, 1.f, "__media_controller_fader"sv);
-    }), nullptr));
+        },
+            1.f, "__media_controller_fader"sv);
+    }),
+                                              nullptr));
 }
 
 void BasicMediaController::setContentSize(const Vec2& contentSize)
@@ -586,8 +591,7 @@ void BasicMediaController::updateControllerState()
         return;
 
     auto state = _mediaPlayer->getState();
-    if (state == MediaPlayer::MediaState::LOADING ||
-        state == MediaPlayer::MediaState::CLOSED ||
+    if (state == MediaPlayer::MediaState::LOADING || state == MediaPlayer::MediaState::CLOSED ||
         state == MediaPlayer::MediaState::ERROR)
     {
         _playButton->setVisible(false);
@@ -882,7 +886,8 @@ MediaPlayer::MediaPlayer()
         pvd->_vrender->setAutoUpdatePS(false);
         this->addProtectedChild(pvd->_vrender);
         /// setup media event callback
-        pvd->_engine->setCallbacks([this, pvd](MEMediaEventType event) {
+        pvd->_engine->setCallbacks(
+            [this, pvd](MEMediaEventType event) {
             switch (event)
             {
             case MEMediaEventType::Playing:
@@ -895,7 +900,7 @@ MediaPlayer::MediaPlayer()
                 break;
 
             case MEMediaEventType::Stopped:
-                onPlayEvent(pvd->_engine->isPlaybackEnded() ? (int) EventType::COMPLETED : (int) EventType::STOPPED);
+                onPlayEvent(pvd->_engine->isPlaybackEnded() ? (int)EventType::COMPLETED : (int)EventType::STOPPED);
                 break;
 
             /* Raised by a media source when a presentation ends. This event signals that all streams in the
@@ -914,7 +919,8 @@ MediaPlayer::MediaPlayer()
                 onPlayEvent((int)EventType::ERROR);
                 break;
             }
-        }, [this, pvd](const ax::MEVideoFrame& frame) {
+        },
+            [this, pvd](const ax::MEVideoFrame& frame) {
             auto pixelFormat       = frame._vpd._PF;
             auto bPixelDescChnaged = !frame._vpd.equals(pvd->_vpixelDesc);
             if (bPixelDescChnaged)
@@ -925,10 +931,17 @@ MediaPlayer::MediaPlayer()
                 pvd->_vtexture = new Texture2D();  // deault is Sampler Filter is: LINEAR
 
                 AX_SAFE_RELEASE_NULL(pvd->_vchromaTexture);
+                AX_SAFE_RELEASE_NULL(pvd->_vchroma2Texture);
                 if (pixelFormat >= MEVideoPixelFormat::YUY2)
                 {  // use separated texture we can set differrent sample filter
                     pvd->_vchromaTexture = new Texture2D();  // Sampler Filter: NEAREST
                     pvd->_vchromaTexture->setAliasTexParameters();
+
+                    if (pixelFormat == MEVideoPixelFormat::I420)
+                    {
+                        pvd->_vchroma2Texture = new Texture2D();  // Sampler Filter: NEAREST
+                        pvd->_vchroma2Texture->setAliasTexParameters();
+                    }
                 }
 
                 auto programManager = ProgramManager::getInstance();
@@ -941,6 +954,9 @@ MediaPlayer::MediaPlayer()
                 case MEVideoPixelFormat::NV12:
                     pvd->_vrender->setProgramState(backend::ProgramType::VIDEO_TEXTURE_NV12);
                     break;
+                case MEVideoPixelFormat::I420:
+                    pvd->_vrender->setProgramState(backend::ProgramType::VIDEO_TEXTURE_I420);
+                    break;
                 default:
                     pvd->_vrender->setProgramState(backend::ProgramType::VIDEO_TEXTURE_RGB32);
                 }
@@ -967,6 +983,18 @@ MediaPlayer::MediaPlayer()
                                                      bufferDim.y >> 1, false, 0);
                 break;
             }
+            case MEVideoPixelFormat::I420:
+            {
+                pvd->_vtexture->updateWithData(frame._dataPointer, bufferDim.x * bufferDim.y, PixelFormat::R8,
+                                               PixelFormat::R8, bufferDim.x, bufferDim.y, false, 0);
+                const auto chromaTexDataSize = (bufferDim.x * bufferDim.y) >> 2;
+                pvd->_vchromaTexture->updateWithData(frame._cbcrDataPointer, chromaTexDataSize, PixelFormat::R8,
+                                                     PixelFormat::R8, bufferDim.x >> 1, bufferDim.y >> 1, false, 0);
+                pvd->_vchroma2Texture->updateWithData(frame._cbcrDataPointer + chromaTexDataSize, chromaTexDataSize,
+                                                      PixelFormat::R8, PixelFormat::R8, bufferDim.x >> 1,
+                                                      bufferDim.y >> 1, false, 0);
+                break;
+            }
             case MEVideoPixelFormat::RGB32:
                 pvd->_vtexture->updateWithData(frame._dataPointer, frame._dataLen, PixelFormat::RGBA8,
                                                PixelFormat::RGBA8, bufferDim.x, bufferDim.y, false, 0);
@@ -989,7 +1017,11 @@ MediaPlayer::MediaPlayer()
                 {
                     auto ps = pvd->_vrender->getProgramState();
                     PrivateVideoDescriptor::updateColorTransform(ps, frame._vpd._fullRange);
+
                     ps->setTexture(ps->getUniformLocation("u_tex1"), 1, pvd->_vchromaTexture->getBackendTexture());
+
+                    if (pixelFormat == MEVideoPixelFormat::I420)
+                        ps->setTexture(ps->getUniformLocation("u_tex2"), 2, pvd->_vchroma2Texture->getBackendTexture());
                 }
 
                 pvd->_scaleDirty = true;
@@ -1016,6 +1048,7 @@ MediaPlayer::~MediaPlayer()
     AX_SAFE_RELEASE(pvd->_vrender);
     AX_SAFE_RELEASE(pvd->_vtexture);
     AX_SAFE_RELEASE(pvd->_vchromaTexture);
+    AX_SAFE_RELEASE(pvd->_vchroma2Texture);
 
     if (g_mediaControlsTexture && g_mediaControlsTexture->getReferenceCount() == 1)
     {