Muxing Android MediaCodec将H264数据包编码为RTMP

时间：2015-02-27 23:12:07

标签： android ffmpeg video-streaming h.264 javacv

我来自一个帖子Encoding H.264 from camera with Android MediaCodec。我的设置非常相似。但是，我尝试使用javacv编写mux编码帧并通过rtmp广播它们。

RtmpClient.java 

...
private volatile BlockingQueue<byte[]> mFrameQueue = new LinkedBlockingQueue<byte[]>(MAXIMUM_VIDEO_FRAME_BACKLOG);
...
private void startStream() throws FrameRecorder.Exception, IOException {
    if (TextUtils.isEmpty(mDestination)) {
        throw new IllegalStateException("Cannot start RtmpClient without destination");
    }

    if (mCamera == null) {
        throw new IllegalStateException("Cannot start RtmpClient without camera.");
    }

    Camera.Parameters cameraParams = mCamera.getParameters();

    mRecorder = new FFmpegFrameRecorder(
            mDestination,
            mVideoQuality.resX,
            mVideoQuality.resY,
            (mAudioQuality.channelType.equals(AudioQuality.CHANNEL_TYPE_STEREO) ? 2 : 1));

    mRecorder.setFormat("flv");

    mRecorder.setFrameRate(mVideoQuality.frameRate);
    mRecorder.setVideoBitrate(mVideoQuality.bitRate);
    mRecorder.setVideoCodec(avcodec.AV_CODEC_ID_H264);

    mRecorder.setSampleRate(mAudioQuality.samplingRate);
    mRecorder.setAudioBitrate(mAudioQuality.bitRate);
    mRecorder.setAudioCodec(avcodec.AV_CODEC_ID_AAC);

    mVideoStream = new VideoStream(mRecorder, mVideoQuality, mFrameQueue, mCamera);
    mAudioStream = new AudioStream(mRecorder, mAudioQuality);

    mRecorder.start();

    // Setup a bufferred preview callback
    setupCameraCallback(mCamera, mRtmpClient, DEFAULT_PREVIEW_CALLBACK_BUFFERS,
            mVideoQuality.resX * mVideoQuality.resY * ImageFormat.getBitsPerPixel(
                    cameraParams.getPreviewFormat())/8);

    try {
        mVideoStream.start();
        mAudioStream.start();
    }
    catch(Exception e) {
        e.printStackTrace();
        stopStream();
    }
}
...
@Override
public void onPreviewFrame(byte[] data, Camera camera) {
    boolean frameQueued = false;

    if (mRecorder == null || data == null) {
        return;
    }

    frameQueued = mFrameQueue.offer(data);

    // return the buffer to be reused - done in videostream
    //camera.addCallbackBuffer(data);
}
...

VideoStream.java 

...
@Override
public void run() {
    try {
        mMediaCodec = MediaCodec.createEncoderByType("video/avc");
        MediaFormat mediaFormat = MediaFormat.createVideoFormat("video/avc", mVideoQuality.resX, mVideoQuality.resY);
        mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, mVideoQuality.bitRate);
        mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, mVideoQuality.frameRate);
        mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar);
        mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1);
        mMediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
        mMediaCodec.start();
    }
    catch(IOException e) {
        e.printStackTrace();
    }

    long startTimestamp = System.currentTimeMillis();
    long frameTimestamp = 0;
    byte[] rawFrame = null;

    try {
        while (!Thread.interrupted()) {
            rawFrame = mFrameQueue.take();

            frameTimestamp = 1000 * (System.currentTimeMillis() - startTimestamp);

            encodeFrame(rawFrame, frameTimestamp);

            // return the buffer to be reused
            mCamera.addCallbackBuffer(rawFrame);
        }
    }
    catch (InterruptedException ignore) {
        // ignore interrup while waiting
    }

    // Clean up video stream allocations
    try {
        mMediaCodec.stop();
        mMediaCodec.release();
        mOutputStream.flush();
        mOutputStream.close();
    } catch (Exception e){
        e.printStackTrace();
    }
}
...
private void encodeFrame(byte[] input, long timestamp) {
    try {
        ByteBuffer[] inputBuffers = mMediaCodec.getInputBuffers();
        ByteBuffer[] outputBuffers = mMediaCodec.getOutputBuffers();

        int inputBufferIndex = mMediaCodec.dequeueInputBuffer(0);

        if (inputBufferIndex >= 0) {
            ByteBuffer inputBuffer = inputBuffers[inputBufferIndex];
            inputBuffer.clear();
            inputBuffer.put(input);
            mMediaCodec.queueInputBuffer(inputBufferIndex, 0, input.length, timestamp, 0);
        }

        MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();

        int outputBufferIndex = mMediaCodec.dequeueOutputBuffer(bufferInfo, 0);

        if (outputBufferIndex >= 0) {
            while (outputBufferIndex >= 0) {
                ByteBuffer outputBuffer = outputBuffers[outputBufferIndex];

                // Should this be a direct byte buffer?
                byte[] outData = new byte[bufferInfo.size - bufferInfo.offset];
                outputBuffer.get(outData);

                mFrameRecorder.record(outData, bufferInfo.offset, outData.length, timestamp);

                mMediaCodec.releaseOutputBuffer(outputBufferIndex, false);
                outputBufferIndex = mMediaCodec.dequeueOutputBuffer(bufferInfo, 0);
            }
        }
        else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
            outputBuffers = mMediaCodec.getOutputBuffers();
        } else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
            // ignore for now
        }
    } catch (Throwable t) {
        t.printStackTrace();
    }

}
...

FFmpegFrameRecorder.java 

...
// Hackish codec copy frame recording function
public boolean record(byte[] encodedData, int offset, int length, long frameCount) throws Exception {
    int ret;

    if (encodedData == null) {
        return false;
    }

    av_init_packet(video_pkt);

    // this is why i wondered whether I should get outputbuffer data into direct byte buffer
    video_outbuf.put(encodedData, 0, encodedData.length);

    video_pkt.data(video_outbuf);
    video_pkt.size(video_outbuf_size);

    video_pkt.pts(frameCount);
    video_pkt.dts(frameCount);

    video_pkt.stream_index(video_st.index());

    synchronized (oc) {
        /* write the compressed frame in the media file */
        if (interleaved && audio_st != null) {
            if ((ret = av_interleaved_write_frame(oc, video_pkt)) < 0) {
                throw new Exception("av_interleaved_write_frame() error " + ret + " while writing interleaved video frame.");
            }
        } else {
            if ((ret = av_write_frame(oc, video_pkt)) < 0) {
                throw new Exception("av_write_frame() error " + ret + " while writing video frame.");
            }
        }
    }
    return (video_pkt.flags() & AV_PKT_FLAG_KEY) == 1;
}
...

当我尝试流式传输视频并在其上运行ffprobe时，我得到以下输出：

ffprobe version 2.5.3 Copyright (c) 2007-2015 the FFmpeg developers
  built on Jan 19 2015 12:56:57 with gcc 4.1.2 (GCC) 20080704 (Red Hat 4.1.2-55)
  configuration: --prefix=/usr --bindir=/usr/bin --datadir=/usr/share/ffmpeg --incdir=/usr/include/ffmpeg --libdir=/usr/lib64 --mandir=/usr/share/man --arch=x86_64 --optflags='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=generic' --enable-bzlib --disable-crystalhd --enable-libass --enable-libdc1394 --enable-libfaac --enable-nonfree --disable-indev=jack --enable-libfreetype --enable-libgsm --enable-libmp3lame --enable-openal --enable-libopencv --enable-libopenjpeg --enable-libopus --enable-librtmp --enable-libtheora --enable-libvorbis --enable-libvpx --enable-libx264 --enable-libxvid --enable-x11grab --enable-avfilter --enable-avresample --enable-postproc --enable-pthreads --disable-static --enable-shared --enable-gpl --disable-debug --disable-stripping --enable-libcaca --shlibdir=/usr/lib64 --enable-runtime-cpudetect
  libavutil      54. 15.100 / 54. 15.100
  libavcodec     56. 13.100 / 56. 13.100
  libavformat    56. 15.102 / 56. 15.102
  libavdevice    56.  3.100 / 56.  3.100
  libavfilter     5.  2.103 /  5.  2.103
  libavresample   2.  1.  0 /  2.  1.  0
  libswscale      3.  1.101 /  3.  1.101
  libswresample   1.  1.100 /  1.  1.100
  libpostproc    53.  3.100 / 53.  3.100
Metadata:
  Server                NGINX RTMP (github.com/arut/nginx-rtmp-module)
  width                 320.00
  height                240.00
  displayWidth          320.00
  displayHeight         240.00
  duration              0.00
  framerate             0.00
  fps                   0.00
  videodatarate         261.00
  videocodecid          7.00
  audiodatarate         62.00
  audiocodecid          10.00
  profile
  level
[live_flv @ 0x1edb0820] Could not find codec parameters for stream 0 (Video: none, none, 267 kb/s): unknown codec
Consider increasing the value for the 'analyzeduration' and 'probesize' options
Input #0, live_flv, from 'rtmp://<server>/input/<stream id>':
  Metadata:
    Server          : NGINX RTMP (github.com/arut/nginx-rtmp-module)
    displayWidth    : 320
    displayHeight   : 240
    fps             : 0
    profile         :
    level           :
  Duration: 00:00:00.00, start: 16.768000, bitrate: N/A
    Stream #0:0: Video: none, none, 267 kb/s, 1k tbr, 1k tbn, 1k tbc
    Stream #0:1: Audio: aac (LC), 16000 Hz, mono, fltp, 63 kb/s
Unsupported codec with id 0 for input stream 0

无论如何，我不是H264或视频编码方面的专家。我知道从M​​ediaCodec出来的编码帧包含SPS NAL，PPS NAL和帧NAL单元。我还将MediaCodec输出写入文件并能够播放（我必须指定格式和帧速率，否则播放速度太快）。

我的假设是事情应该有效（看我知道多少:)）。知道SPS和PPS被写出来，解码器应该足够了解。然而，ffprobe无法识别编解码器，fps和其他视频信息。我是否需要将包标志信息传递给FFmpegFrameRecorder.java:record（）函数？或者我应该使用直接缓冲？任何建议将不胜感激！我应该提示一下。

PS：我知道有些编解码器使用Planar和其他SemiPlanar颜色格式。如果我超越这个，这种区别将会在以后出现。另外，我没有使用Surface到MediaCodec方式，因为我需要支持API 17，它需要比这条路线更多的更改，我认为这有助于我理解更基本的流程。阿甘，我感谢任何建议。如果需要澄清的话，请告诉我。

更新＃1

因此，我做了更多测试，我看到我的编码器输出以下帧：

000000016742800DDA0507E806D0A1350000000168CE06E2
0000000165B840A6F1E7F0EA24000AE73BEB5F51CC7000233A84240...
0000000141E2031364E387FD4F9BB3D67F51CC7000279B9F9CFE811...
0000000141E40304423FFFFF0B7867F89FAFFFFFFFFFFCBE8EF25E6...
0000000141E602899A3512EF8AEAD1379F0650CC3F905131504F839...
...

第一帧包含SPS和PPS。从我能够看到的，这些只传输一次。其余的是NAL类型1和5.因此，我的假设是，为了ffprobe不仅在流开始时看到流信息，我应该捕获SPS和PPS帧并在一定数量的帧之后定期重新发送它们，或者也许在每个I帧之前。你觉得怎么样？

更新＃2

无法验证我是否正在成功编写帧。在尝试回读写入的数据包之后，我无法验证写入的字节。奇怪的是，在成功写入IPL图像和流式传输之后，我也无法在avcodec_encode_video2之后打印出编码数据包的字节。击中官方死胡同。

1 个答案:

答案 0 :(得分：0)

据我所知，您不需要多路复用视频和音频流。发送ANNOUNCE消息时，还指定要向哪些端口发送音频和视频流。您需要单独打包并使用RTP发送。有关详细信息，请参阅维基百科。 https://en.wikipedia.org/wiki/Real_Time_Streaming_Protocol

相关问题
最新问题