使用MediaCodec解码h.264流,dequeueOutputBuffer始终返回-1

时间:2014-12-27 04:46:41

标签: android ffmpeg h.264 live-streaming mediacodec

我正在尝试使用MediaCodec API通过ffmpeg解码来自PC的实时流屏幕捕获。

对于发件人(PC ffmpeg)

我使用此命令

ffmpeg -re -f gdigrab -s 1920x1080 -threads 4 -i desktop -vcodec libx264 -pix_fmt yuv420p -tune zerolatency -profile:v baseline -flags global_header -s 1280x720 -an -f rtp rtp://192.168.1.6:1234

和输出看起来像这样

Output #0, rtp, to 'rtp://192.168.1.6:1234':
  Metadata:
    encoder         : Lavf56.15.104
    Stream #0:0: Video: h264 (libx264), yuv420p, 1280x720, q=-1--1, 29.97 fps, 90k tbn, 29.97 tbc
Metadata:
  encoder         : Lavc56.14.100 libx264
Stream mapping:
  Stream #0:0 -> #0:0 (bmp (native) -> h264 (libx264))
SDP:
v=0
o=- 0 0 IN IP4 127.0.0.1
s=No Name
c=IN IP4 192.168.1.6
t=0 0
a=tool:libavformat 56.15.104
m=video 1234 RTP/AVP 96
a=rtpmap:96 H264/90000
a=fmtp:96 packetization-mode=1; sprop-parameter-sets=Z0LAH9kAUAW6EAAAPpAADqYI8YMkgA==,aMuDyyA=; profile-level-id=42C01F

Press [q] to stop, [?] for help
frame=   19 fps=0.0 q=17.0 size=     141kB time=00:00:00.63 bitrate=1826.0kbits/
frame=   34 fps= 32 q=17.0 size=     164kB time=00:00:01.13 bitrate=1181.5kbits/
frame=   50 fps= 32 q=18.0 size=     173kB time=00:00:01.66 bitrate= 850.9kbits/

对于Receiver(Android MediaCodec)

我用surface创建了活动并实现了SurfaceHolder.Callback

在surfaceChanged中

@Override
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
    Log.i("sss", "surfaceChanged");
    if( playerThread == null ) {
        playerThread = new PlayerThread(holder.getSurface());
        playerThread.start();
    }

}

对于PlayerThread

class PlayerThread extends Thread {

    MediaCodec decoder;
    Surface surface;

    public PlayerThread(Surface surface) {
        this.surface = surface;
    }

    @Override
    public void run() {
        running = true;
        try {
            MediaFormat format = MediaFormat.createVideoFormat("video/avc", 1280, 720);
            byte[] header = new byte[] {0,0,0,1};
            byte[] sps = Base64.decode("Z0LAH9kAUAW6EAAAPpAADqYI8YMkgA==", Base64.DEFAULT);
            byte[] pps = Base64.decode("aMuDyyA=", Base64.DEFAULT);

            byte[] header_sps = new byte[sps.length + header.length];
            System.arraycopy(header,0,header_sps,0,header.length);
            System.arraycopy(sps,0,header_sps,header.length, sps.length);

            byte[] header_pps = new byte[pps.length + header.length];
            System.arraycopy(header,0, header_pps, 0, header.length);
            System.arraycopy(pps, 0, header_pps, header.length, pps.length);

            format.setByteBuffer("csd-0", ByteBuffer.wrap(header_sps));
            format.setByteBuffer("csd-1", ByteBuffer.wrap(header_pps));
            format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 1280 * 720);
//          format.setInteger("durationUs", 63446722);
//          format.setByteBuffer("csd-2", ByteBuffer.wrap((hexStringToByteArray("42C01E"))));                      
//          format.setInteger(MediaFormat.KEY_COLOR_FORMAT ,MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar);
            Log.i("sss", "Format = " + format);

            try {
                decoder = MediaCodec.createDecoderByType("video/avc");
                decoder.configure(format, surface, null, 0);
                decoder.start();

            } catch (IOException ioEx) {
                ioEx.printStackTrace();
            }

            DatagramSocket socket = new DatagramSocket(1234);
            byte[] bytes = new byte[4096];
            DatagramPacket packet = new DatagramPacket(bytes, bytes.length);

            byte[] data;

            ByteBuffer[] inputBuffers;
            ByteBuffer[] outputBuffers;

            ByteBuffer inputBuffer;
            ByteBuffer outputBuffer;

            MediaCodec.BufferInfo bufferInfo;

            bufferInfo = new MediaCodec.BufferInfo();
            int inputBufferIndex;
            int outputBufferIndex;
            byte[] outData;

            inputBuffers = decoder.getInputBuffers();
            outputBuffers = decoder.getOutputBuffers();

            int minusCount = 0;
            byte[] prevData = new byte[65535];
            List<byte[]> playLoads = new ArrayList<>();
            int playloadSize = 0;
            while (true) {
                try {
                    socket.receive(packet);
                    data = new byte[packet.getLength()];
                    System.arraycopy(packet.getData(), packet.getOffset(), data, 0, packet.getLength());

                inputBufferIndex = decoder.dequeueInputBuffer(-1);
                    Log.i("sss", "inputBufferIndex = " + inputBufferIndex);
                if (inputBufferIndex >= 0)
                {
                    inputBuffer = inputBuffers[inputBufferIndex];
                    inputBuffer.clear();

                    inputBuffer.put(data);


                    decoder.queueInputBuffer(inputBufferIndex, 0, data.length, 0, 0);
//                  decoder.flush();
                }

                outputBufferIndex = decoder.dequeueOutputBuffer(bufferInfo, 10000);
                Log.i("sss", "outputBufferIndex = " + outputBufferIndex);

                while (outputBufferIndex >= 0)
                {
                    outputBuffer = outputBuffers[outputBufferIndex];

                    outputBuffer.position(bufferInfo.offset);
                    outputBuffer.limit(bufferInfo.offset + bufferInfo.size);

                    outData = new byte[bufferInfo.size];
                    outputBuffer.get(outData);


                    decoder.releaseOutputBuffer(outputBufferIndex, false);
                    outputBufferIndex = decoder.dequeueOutputBuffer(bufferInfo, 0);

                }

                } catch (SocketTimeoutException e) {
                    Log.d("thread", "timeout");
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

我认为来自ffmpeg的流不是问题,因为我可以通过sdp文件从mxPlayer打开它。 如果我将此流传递到本地RTSP服务器(通过VLC),那么我使用MediaPlayer获取RTSP流,它可以工作但速度很慢。

在我查看数据包后,我意识到了

  • 前四个字节是标题和序列号
  • 接下来的四个字节是TimeStamp
  • 接下来的四个字节是源标识符

因此,我将前12个字节切掉并将数据包与相同的TimeStamp组合在一起。然后把它放在像这样的缓冲区

在收到数据包后的while(true)

                Log.i("sss", "Received = " + data.length + " bytes");
                Log.i("sss","prev " + prevData.length + " bytes = " + getBytesStr(prevData));
                Log.i("sss","data " + data.length + " bytes = " + getBytesStr(data));

                        if(data[4] == prevData[4] && data[5] == prevData[5] && data[6] == prevData[6] && data[7] == prevData[7]){
                            byte[] playload = new byte[prevData.length -12];
                            System.arraycopy(prevData,12,playload, 0, prevData.length-12);
                            playLoads.add(playload);
                            playloadSize += playload.length;
                            Log.i("sss", "Same timeStamp playload " + playload.length + " bytes = " + getBytesStr(playload));
                        } else {
                            if(playLoads.size() > 0){
                                byte[] playload = new byte[prevData.length -12];
                                System.arraycopy(prevData,12,playload, 0, prevData.length-12);
                                playLoads.add(playload);
                                playloadSize += playload.length;
                                Log.i("sss", "last playload " + playload.length + " bytes = " + getBytesStr(playload));

                                inputBufferIndex = decoder.dequeueInputBuffer(-1);
                                if (inputBufferIndex >= 0){
                                    inputBuffer = inputBuffers[inputBufferIndex];
                                    inputBuffer.clear();
                                    byte[] allPlayload = new byte[playloadSize];
                                    int curLength = 0;
                                    for(byte[] playLoad:playLoads){
                                        System.arraycopy(playLoad,0,allPlayload, curLength, playLoad.length);
                                        curLength += playLoad.length;
                                    }
                                    Log.i("sss", "diff timeStamp AlllayLoad " + allPlayload.length + "bytes = " + getBytesStr(allPlayload));
                                    inputBuffer.put(allPlayload);

                                    decoder.queueInputBuffer(inputBufferIndex, 0, data.length, 0, 0);
                                    decoder.flush();
                                }

                                bufferInfo = new MediaCodec.BufferInfo();
                                outputBufferIndex = decoder.dequeueOutputBuffer(bufferInfo, 10000);
                                if(outputBufferIndex!= -1)
                                    Log.i("sss", "outputBufferIndex = " + outputBufferIndex);

                                playLoads = new ArrayList<>();
                                prevData = new byte[65535];
                                playloadSize = 0;
                            }

                        }

                    prevData = data.clone();

outputBufferIndex仍返回-1

如果我将timeoutUS从10000更改为-1,它永远不会转到下一行

我已经搜索了一周,但仍然没有运气T_T

为什么dequeueOutputBuffer总是返回-1?

我的代码有什么问题?

你能否正确优化我的代码才能正常工作?

感谢您的帮助。

修改#1

谢谢@mstorsjo指导我打包,我找到了有用的信息

How to process raw UDP packets so that they can be decoded by a decoder filter in a directshow source filter

然后我编辑了下面的代码

if((data[12] & 0x1f) == 28){
   if((data[13] & 0x80) == 0x80){ //found start bit
      inputBufferIndex = decoder.dequeueInputBuffer(-1);
      if (inputBufferIndex >= 0){
         inputBuffer = inputBuffers[inputBufferIndex];
         inputBuffer.clear();
         byte result = (byte)((bytes[12] & 0xe0) + (bytes[13] & 0x1f));
         inputBuffer.put(new byte[] {0,0,1});
         inputBuffer.put(result);
         inputBuffer.put(data,14, data.length-14);
      }

   } else if((data[13] &0x40) == 0x40){ //found stop bit
      inputBuffer.put(data, 14, data.length -14);
      decoder.queueInputBuffer(inputBufferIndex, 0, data.length, 0, 0);
      bufferInfo = new MediaCodec.BufferInfo();
      outputBufferIndex = decoder.dequeueOutputBuffer(bufferInfo, 10000);

      switch(outputBufferIndex)
      {
         case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
            outputBuffers = decoder.getOutputBuffers();
            Log.w("sss", "Output Buffers Changed");
            break;
         case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
            Log.w("sss", "Output Format Changed");
            MediaFormat newFormat = decoder.getOutputFormat();
            Log.i("sss","New format : " + newFormat);

            break;
         case MediaCodec.INFO_TRY_AGAIN_LATER:
            Log.w("sss", "Try Again Later");
            break;
         default:
            outputBuffer = outputBuffers[outputBufferIndex];
            outputBuffer.position(bufferInfo.offset);
            outputBuffer.limit(bufferInfo.offset + bufferInfo.size);
            decoder.releaseOutputBuffer(outputBufferIndex, true);

      }
   } else {
      inputBuffer.put(data, 14, data.length -14);
   }
 }

现在我可以看到一些图片,但大部分屏幕都是灰色的

我接下来应该做什么?

谢谢。

2 个答案:

答案 0 :(得分:1)

您不能只丢弃RTP标头并假装其余数据包是正常的H264帧 - 事实并非如此。有关将H264打包成RTP时使用的格式的说明,请参阅RFC 6184。您需要撤消此打包以将数据恢复为普通解码器可以处理的格式。您可以查看libav / ffmpeg中的libavformat/rtpdec_h264.c以获取有关如何执行此操作的示例。

答案 1 :(得分:0)

这可能会迟到,但我可以看到两个可能的问题。

1)你只看NAL 28型(FU-A)的NAL单位,但是ffmpeg正在发送类型为1,24和28的NAL单位.24种NAL单位可以忽略而没有风险,但是类型1 NAL单位不能忽略(他们有NRI> 0)。

2)rtp流不一定按照发送顺序到达。因此,可能以错误的顺序重建帧。要确保正确的顺序,您必须查看rtp标头中的时间戳。

我找到一个好的库,这是Anroid Streaming Client。您需要稍微修改它以在MediaFormat中使用正确的csd-0 / csd-1,并将输出缓冲区设置为任意曲面而不是SurfaceView中的一个。