Question

我几天前完成了自己的Android流媒体库（API 18）。它可以通过RTMP读取H.264并通过SurfaceTexture（GL_TEXTURE_EXTERNAL_OES目标）+ QSGNode输出实时解码（非常感谢Bog＆＃39;来自KDAB的Dan Vatra以及如何使用结合这个），它也支持Camera流媒体。 Party开始很棒，但是......我不能同时创建3个输出，因为它们都渲染到相同的纹理。我看到了什么：

首先输出必须是相机输出。
第二 - 从第一个RTMP解码H.264。
第三 - 从另一个RTMP解码H.264（ffmpeg电影）。

尝试通过FBO渲染，使用GL_TEXTURE0代替GL_TEXTURE1，合并GL_TEXTURE0/1/2/3和许多事情......没有效果。一次只有一个纹理或完全空白四边形。

库有太多代码，为了简单起见，我将仅从KDAB发布一些来源（https://github.com/KDAB/android/tree/master/examples/qtsurfacetexture）。它有同样的错误。

main.qml - 针对2部不同电影的两个SurfaceTexture的QML来源：

import QtQuick 2.5
import QtQuick.Controls 1.4
import QtQuick.Layouts 1.3
import SurfaceTexture 1.0

ApplicationWindow {
    visible: true
    visibility: "FullScreen"
    width: 640
    height: 480
    title: qsTr("SurfaceTexture example")

    ColumnLayout {
        anchors.fill: parent

        SurfaceTexture {
            id: videoItem
            Layout.alignment: Qt.AlignCenter
            Layout.preferredWidth: parent.width
            Layout.preferredHeight: parent.height * 0.25

            // Set media player's video out
            Component.onCompleted: _mediaPlayer.videoOut = videoItem;

            MouseArea {
                anchors.fill: parent
                onClicked: _mediaPlayer.playFile("/storage/emulated/0/Movies/TLOTR/TLOTR1_TFOTR.mp4");
            }
        }

        SurfaceTexture {
            id: videoItem0
            Layout.alignment: Qt.AlignCenter
            Layout.preferredWidth: parent.width
            Layout.preferredHeight: parent.height * 0.25

            // Set media player's video out
            Component.onCompleted: _mediaPlayer.videoOut = videoItem0;

            MouseArea {
                anchors.fill: parent
                onClicked: _mediaPlayer.playFile("/storage/emulated/0/Movies/TLOTR/TLOTR2_TTT.mp4");
            }
        }
    }
}

qandroidmediaplayer.cpp - 使用JNI的Android MediaPlayer实现：

#include "qandroidmediaplayer.h"
#include "qsurfacetexture.h"

#include <QAndroidJniEnvironment>

QAndroidMediaPlayer::QAndroidMediaPlayer(QObject *parent)
    : QObject(parent)
    , m_mediaPlayer("android/media/MediaPlayer")
{
}

QAndroidMediaPlayer::~QAndroidMediaPlayer()
{
    QAndroidJniEnvironment env;
    m_mediaPlayer.callMethod<void>("stop");
    m_mediaPlayer.callMethod<void>("reset");
    m_mediaPlayer.callMethod<void>("release");
}

QSurfaceTexture *QAndroidMediaPlayer::videoOut() const
{
    return m_videoOut;
}

void QAndroidMediaPlayer::setVideoOut(QSurfaceTexture *videoOut)
{
    if (m_videoOut == videoOut)
        return;
    m_videoOut = videoOut;

    // Create a new Surface object from our SurfaceTexture
    QAndroidJniObject surface("android/view/Surface",
                              "(Landroid/graphics/SurfaceTexture;)V",
                               videoOut->surfaceTexture().object());

    // Set the new surface to m_mediaPlayer object
    m_mediaPlayer.callMethod<void>("setSurface", "(Landroid/view/Surface;)V",
                                   surface.object());

    emit videoOutChanged();
}

void QAndroidMediaPlayer::playFile(const QString &file)
{
    QAndroidJniEnvironment env;
    m_mediaPlayer.callMethod<void>("stop"); // try to stop the media player.
    m_mediaPlayer.callMethod<void>("reset"); // try to reset the media player.

    // http://developer.android.com/reference/android/media/MediaPlayer.html#setDataSource(java.lang.String) - the path of the file, or the http/rtsp URL of the stream you want to play.
    m_mediaPlayer.callMethod<void>("setDataSource", "(Ljava/lang/String;)V", QAndroidJniObject::fromString(file).object());

    // prepare media player
    m_mediaPlayer.callMethod<void>("prepare");

    // start playing
    m_mediaPlayer.callMethod<void>("start");
}

qsurfacetexture.cpp - SurfaceTexture QQuickItem。使用QSGNode进行渲染：

#include "qsurfacetexture.h"

#include <QAndroidJniEnvironment>
#include <QSGGeometryNode>
#include <QSGSimpleMaterialShader>

struct State {
    // the texture transform matrix
    QMatrix4x4 uSTMatrix;

    int compare(const State *other) const
    {
        return uSTMatrix == other->uSTMatrix ? 0 : -1;
    }
};

class SurfaceTextureShader : QSGSimpleMaterialShader<State>
{
    QSG_DECLARE_SIMPLE_COMPARABLE_SHADER(SurfaceTextureShader, State)
public:
    // vertex & fragment shaders are shamelessly "stolen" from MyGLSurfaceView.java :)
    const char *vertexShader() const {
        return
                "uniform mat4 qt_Matrix;                            \n"
                "uniform mat4 uSTMatrix;                            \n"
                "attribute vec4 aPosition;                          \n"
                "attribute vec4 aTextureCoord;                      \n"
                "varying vec2 vTextureCoord;                        \n"
                "void main() {                                      \n"
                "  gl_Position = qt_Matrix * aPosition;             \n"
                "  vTextureCoord = (uSTMatrix * aTextureCoord).xy;  \n"
                "}";
    }

    const char *fragmentShader() const {
        return
                "#extension GL_OES_EGL_image_external : require                     \n"
                "precision mediump float;                                           \n"
                "varying vec2 vTextureCoord;                                        \n"
                "uniform lowp float qt_Opacity;                                     \n"
                "uniform samplerExternalOES sTexture;                               \n"
                "void main() {                                                      \n"
                "  gl_FragColor = texture2D(sTexture, vTextureCoord) * qt_Opacity;  \n"
                "}";
    }

    QList<QByteArray> attributes() const
    {
        return QList<QByteArray>() << "aPosition" << "aTextureCoord";
    }

    void updateState(const State *state, const State *)
    {
        program()->setUniformValue(m_uSTMatrixLoc, state->uSTMatrix);
    }

    void resolveUniforms()
    {
        m_uSTMatrixLoc = program()->uniformLocation("uSTMatrix");
        program()->setUniformValue("sTexture", 0); // we need to set the texture once
    }

private:
    int m_uSTMatrixLoc;
};

class SurfaceTextureNode : public QSGGeometryNode
{
public:
    SurfaceTextureNode(const QAndroidJniObject &surfaceTexture, GLuint textureId)
        : QSGGeometryNode()
        , m_surfaceTexture(surfaceTexture)
        , m_geometry(QSGGeometry::defaultAttributes_TexturedPoint2D(), 4)
        , m_textureId(textureId)
    {
        // we're going to use "preprocess" method to update the texture image
        // and to get the new matrix.
        setFlag(UsePreprocess);

        setGeometry(&m_geometry);

        // Create and set our SurfaceTextureShader
        QSGSimpleMaterial<State> *material = SurfaceTextureShader::createMaterial();
        material->setFlag(QSGMaterial::Blending, false);
        setMaterial(material);
        setFlag(OwnsMaterial);

        // We're going to get the transform matrix for every frame
        // so, let's create the array once
        QAndroidJniEnvironment env;
        jfloatArray array = env->NewFloatArray(16);
        m_uSTMatrixArray = jfloatArray(env->NewGlobalRef(array));
        env->DeleteLocalRef(array);
    }

    ~SurfaceTextureNode()
    {
        // delete the global reference, now the gc is free to free it
        QAndroidJniEnvironment()->DeleteGlobalRef(m_uSTMatrixArray);
    }

    // QSGNode interface
    void preprocess() override;

private:
    QAndroidJniObject m_surfaceTexture;
    QSGGeometry m_geometry;
    jfloatArray m_uSTMatrixArray = nullptr;
    GLuint m_textureId;
};

void SurfaceTextureNode::preprocess()
{
    QSGSimpleMaterial<State> *mat = static_cast<QSGSimpleMaterial<State> *>(material());
    if (!mat)
        return;

    // update the texture content
    m_surfaceTexture.callMethod<void>("updateTexImage");

    // get the new texture transform matrix
    m_surfaceTexture.callMethod<void>("getTransformMatrix", "([F)V", m_uSTMatrixArray);
    QAndroidJniEnvironment env;
    env->GetFloatArrayRegion(m_uSTMatrixArray, 0, 16, mat->state()->uSTMatrix.data());

    // Activate and bind our texture
    glActiveTexture(GL_TEXTURE0);
    glBindTexture(GL_TEXTURE_EXTERNAL_OES, m_textureId);
}

extern "C" void Java_com_kdab_android_SurfaceTextureListener_frameAvailable(JNIEnv */*env*/, jobject /*thiz*/, jlong ptr, jobject /*surfaceTexture*/)
{
    // a new frame was decoded, let's update our item
    QMetaObject::invokeMethod(reinterpret_cast<QSurfaceTexture*>(ptr), "update", Qt::QueuedConnection);
}

QSurfaceTexture::QSurfaceTexture(QQuickItem *parent)
    : QQuickItem(parent)
{
    setFlags(ItemHasContents);
}

QSurfaceTexture::~QSurfaceTexture()
{
    // Delete our texture
    if (m_textureId) {
        glBindTexture(GL_TEXTURE_EXTERNAL_OES, 0);
        glDeleteTextures(1, &m_textureId);
    }
}

QSGNode *QSurfaceTexture::updatePaintNode(QSGNode *n, QQuickItem::UpdatePaintNodeData *)
{
    SurfaceTextureNode *node = static_cast<SurfaceTextureNode *>(n);
    if (!node) {
        // Create texture
        glGenTextures(1, &m_textureId);
        glBindTexture(GL_TEXTURE_EXTERNAL_OES, m_textureId);

        // Can't do mipmapping with camera source
        glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
        glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR);

        // Clamp to edge is the only option
        glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
        glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);

        // Create surface texture Java object
        m_surfaceTexture = QAndroidJniObject("android/graphics/SurfaceTexture", "(I)V", m_textureId);

        // We need to setOnFrameAvailableListener, to be notify when a new frame was decoded
        // and is ready to be displayed. Check android/src/com/kdab/android/SurfaceTextureListener.java
        // file for implementation details.
        m_surfaceTexture.callMethod<void>("setOnFrameAvailableListener",
                                          "(Landroid/graphics/SurfaceTexture$OnFrameAvailableListener;)V",
                                          QAndroidJniObject("com/kdab/android/SurfaceTextureListener",
                                                            "(J)V", jlong(this)).object());

        // Create our SurfaceTextureNode
        node = new SurfaceTextureNode(m_surfaceTexture, m_textureId);
    }

    // flip vertical
    QRectF rect(boundingRect());
    float tmp = rect.top();
    rect.setTop(rect.bottom());
    rect.setBottom(tmp);

    QSGGeometry::updateTexturedRectGeometry(node->geometry(), rect, QRectF(0, 0, 1, 1));
    node->markDirty(QSGNode::DirtyGeometry | QSGNode::DirtyMaterial);
    return node;
}

然后在应用程序中点击SurfaceTexture后，我们可以看到下一张奇怪的图片：

两部不同的电影，但一次只能绘制一个纹理。此外，第二个输出垂直镜像。如果我说，我一次只能呈现一个GL_TEXTURE_EXTERNAL_OES，那么我会是真的吗？如果可能有超过1个纹理，请给我一些想法，我必须在代码中修复。

集成QtQuick和GL_TEXTURE_EXTERNAL_OES

0 个答案: