使用FFmpeg从音频和视频文件复用

时间:2013-05-27 08:02:48

标签: c++ audio ffmpeg video-encoding

我正在学习如何从this示例创建MP4视频。问题是该示例演示了动态生成的一些虚拟源数据的音频编码。我需要对文件中的音频进行编码。我已经检查过许多示例和大多数示例显示相同或仅单独的音频编码。 在我的试错过程中,我对音频和视频帧使用相同的AVFormatContext。我不确定这是否正确,或者我是否应该有2个单独的上下文?到目前为止,我的视频编码还可以,但是音频流失败了因为AVPacket无法找到正确的音频流索引。 以下是我设置音频流的方法:

  void open_audio(AVFormatContext *oc, AVCodec **codec, AVStream **st ,enum AVCodecID codec_id){

    //    AVCodecContext *c;
    int ret;
    //    c = st->codec;

    *codec = avcodec_find_encoder(codec_id);
    if (!(*codec)) {
        fprintf(stderr, "Could not find encoder for '%s'\n",avcodec_get_name(codec_id));

    }
    /* open it */



    if(avformat_open_input(&oc,_audioInName.c_str(),NULL,NULL) !=0){

        Msg::PrintErrorMsg("Error opening audio file");

    }


    AVStream* audioStream = NULL;

    // Find the audio stream (some container files can have multiple streams in them)

    for (uint32_t i = 0; i < oc->nb_streams; ++i)

    {

        if (oc->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO)

        {

            audioStream = oc->streams[i];

            break;

        }

    }

    if (audioStream == NULL)
    {
        Msg::PrintErrorMsg("Could not find any audio stream in the file");

    }
    *st =audioStream;

    AVCodecContext *c  = audioStream->codec;
    c->codec = *codec;//avcodec_find_decoder(c->codec_id);
    audioStream->id = 1;
    c->sample_fmt  = AV_SAMPLE_FMT_S16;
    c->bit_rate    = 64000;
    c->sample_rate = 44100;
    c->channels    = 1;

    if (oc->oformat->flags & AVFMT_GLOBALHEADER){
        c->flags |= CODEC_FLAG_GLOBAL_HEADER;

    }

    if (c->codec == NULL)
    {
        Msg::PrintErrorMsg("Couldn't find a proper decoder");

    }

    ret = avcodec_open2(c, *codec, NULL);
    if (ret < 0) {

        Msg::PrintErrorMsg("Could not open audio codec\n");

    }

}

此处“oc”与用于初始化视频流的上下文相同。

然后我尝试写这样的音频帧:

  void write_audio_frame(AVFormatContext *oc, AVStream *st){
    AVCodecContext *c;
    AVPacket pkt = { 0 }; // data and size must be 0;
    AVFrame *frame = avcodec_alloc_frame();
    int got_packet, ret;
    av_init_packet(&pkt);
    c = st->codec;
    /////
    //  get_audio_frame(samples, audio_input_frame_size, c->channels);

    ////Read the packet:
    while(av_read_frame(oc,&pkt) == 0 ){

        if(pkt.stream_index ==st->index){

        // Try to decode the packet into a frame
        int frameFinished = 0;
        avcodec_decode_audio4(c, frame, &frameFinished, &pkt);

        // Some frames rely on multiple packets, so we have to make sure the frame is finished before
        // we can use it
        if (frameFinished){
            assert(frameFinished);
            ret = avcodec_encode_audio2(c, &pkt, frame, &got_packet);
            if (ret < 0) {
                Msg::PrintErrorMsg("Error encoding audio frame\n");

            }
            if (!got_packet){
                printf("failed to aquire packet");
            }
            pkt.stream_index = st->index;
            /* Write the compressed frame to the media file. */
            ret = av_interleaved_write_frame(oc, &pkt);
            if (ret != 0) {

                Msg::PrintErrorMsg("Error while writing audio frame.");
            }

          }
        }

       }
    }
    av_free_packet(&pkt);
    avcodec_free_frame(&frame);
}

问题是我从未传递过这样的说法:“if(pkt.stream_index == st-&gt; index)”。数据包流索引永远不会等于音频流索引。任何人都可以指出我错在哪里?< / p>

更新

我确实设法打开输入音频流进行编码,但我无法将音频和视频流编码为单输出。从我看到PTS和DTS可能是问题的根源。目前我根据多路复用计算pts。例如,但它根本不适用于音频。

以下是我如何使用它:

   while(frame_count < _streamDurationNBFrames-1){

        uint8_t *frameToWrite =_frames.front();


        // Compute current audio and video time. ///

        if (audio_st){
            audio_pts = (double)audioIn_st->pts.val * audioIn_st->time_base.num / audioIn_st->time_base.den;
        }
        else{

            audio_pts = 0.0;
        }
        if (video_st){

            video_pts = (double)video_st->pts.val * video_st->time_base.num /   video_st->time_base.den;

        }else{
            video_pts = 0.0;
        }


        if ((!audio_st || audio_pts >= _streamDuration) && (!video_st || video_pts >= _streamDuration)){

            break;

        }


        if (audio_st && audio_pts < video_pts) {
            av_read_frame(informat, &pkt);//read audio from input stream
             Msg::PrintMsg("Encode audio here...");

          //==================   AUDIO ENCODE HERE   


           outpkt.data = pkt.data;
           outpkt.size = pkt.size;
           outpkt.stream_index = pkt.stream_index;
           outpkt.flags |= AV_PKT_FLAG_KEY;
           outpkt.pts = pkt.pts;
           outpkt.dts =pkt.dts;
           if(av_interleaved_write_frame(oc, &outpkt) < 0)
           {
            Msg::PrintErrorMsg("Fail Audio Write ");
           }
           else
           {
               audio_st->codec->frame_number++;
           }
           av_free_packet(&outpkt);
           av_free_packet(&pkt);



         }else{
          //==================   VIDEO  ENCODE HERE   

            write_video_frame(oc, video_st,frameToWrite);

            frame->pts += av_rescale_q(1, video_st->codec->time_base, video_st->time_base);
         }

        ///at last delete this frame:
        _frames.pop();
        delete frameToWrite; ///deallocate the written frame!
    }

不知何故,一旦我进入音频编码循环,audio_pts永远不会到达video_pts并始终为零:

audio_pts = (double)audio_st->pts.val * audio_st->time_base.num / audio_st->time_base.den; is always zero because  (double)audio_st->pts.val  returns zero.

所以基本上我再次问同样的问题:当音频来自外部文件时如何进行多路复用?

顺便说一句,下面的答案没有用,因为它假设音频和视频流来自同一个文件,而在我的情况下,只有音频来自外部源。

2 个答案:

答案 0 :(得分:8)

您可以使用相同的上下文,不需要两个单独的上下文。如果您正在编码视频和音频。然后,您将首先需要创建视频流,然后创建音频流。但是,如果您只想对音频进行编码,则需要仅创建音频流。 在转码时通常需要if(pkt.stream_index == st-&gt; index),即更改容器格式时。您可以在其中从视频文件中读取帧并写入另一个文件,因此您需要知道帧是来自音频还是视频流。 但是,如果您要获得解码的音频数据包,则需要在执行av_interleaved_write之前在音频数据包中设置正确的流索引。

在您的代码中,您没有设置正确编码所需的音频数据包的pts和dts。

前段时间我写了类似你的类似程序,你可以查看它以供参考。

int VideoClipper::Init(const wxString& filename)
{
    int ret = 0;
    char errbuf[64];

    av_register_all();
    if ((ret = avformat_open_input( &m_informat, filename.mb_str(), 0, 0)) != 0 )
    {
        av_strerror(ret,errbuf,sizeof(errbuf));
        PRINT_VAL("Not able to Open file;; ", errbuf)
        ret = -1;
        return ret;
    }
    else
    {
        PRINT_MSG("Opened File ")
    }

    if ((ret = avformat_find_stream_info(m_informat, 0))< 0 )
    {

        av_strerror(ret,errbuf,sizeof(errbuf));
        PRINT_VAL("Not Able to find stream info:: ", errbuf)
        ret = -1;
        return ret;
    }
    else
    {
        PRINT_MSG("Got stream Info ")
    }

    for(unsigned int i = 0; i<m_informat->nb_streams; i++)
    {
        if(m_informat->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO)
        {

            PRINT_MSG("Found Video Stream ")
            m_in_vid_strm_idx = i;
            m_in_vid_strm = m_informat->streams[i];
        }

        if(m_informat->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO)
        {
            PRINT_MSG("Found Audio Stream ")
            m_in_aud_strm_idx = i;
            m_in_aud_strm = m_informat->streams[i];
        }
    }

    if(m_in_aud_strm_idx == -1 && m_in_vid_strm_idx == -1)
    {
       ret = -1;    
    }

    if(m_informat->duration == AV_NOPTS_VALUE)
    {
        if(m_in_vid_strm_idx != -1 && m_informat->streams[m_in_vid_strm_idx])
        {
            if(m_informat->streams[m_in_vid_strm_idx]->duration != AV_NOPTS_VALUE)
            {
                //m_in_end_time = (m_informat->streams[m_in_vid_strm_idx]->duration)/(AV_TIME_BASE);
                m_in_end_time = (m_informat->streams[m_in_vid_strm_idx]->duration)/(m_informat->streams[m_in_vid_strm_idx]->time_base.den/m_informat->streams[m_in_vid_strm_idx]->time_base.num);

            }

        }
        else if(m_in_aud_strm_idx != -1 && m_informat->streams[m_in_aud_strm_idx])
        {
            if(m_informat->streams[m_in_aud_strm_idx]->duration != AV_NOPTS_VALUE)
            {
                m_in_end_time = (m_informat->streams[m_in_aud_strm_idx]->duration)/(AV_TIME_BASE);
            }
        }
    }
    else
    {
        m_in_end_time = (m_informat->duration)/(AV_TIME_BASE);
    }

    if(m_in_vid_strm_idx != -1 && m_informat->streams[m_in_vid_strm_idx])
    {
        if(m_informat->streams[m_in_vid_strm_idx]->r_frame_rate.num != AV_NOPTS_VALUE && m_informat->streams[m_in_vid_strm_idx]->r_frame_rate.den != 0)
        {
            m_fps =  (m_informat->streams[m_in_vid_strm_idx]->r_frame_rate.num)/ (m_informat->streams[m_in_vid_strm_idx]->r_frame_rate.den);
        }
    }
    else
    {
        m_fps = 25;    
    }
    AVOutputFormat *outfmt = NULL;
    std::string outfile = std::string(filename) + "clip_out.avi";
    outfmt = av_guess_format(NULL,outfile.c_str(),NULL);

    if(outfmt == NULL)
    {
        ret = -1;
        return ret;
    }
    else
    {
        m_outformat = avformat_alloc_context();
        if(m_outformat)
        {
            m_outformat->oformat = outfmt;
            _snprintf(m_outformat->filename, sizeof(m_outformat->filename), "%s", outfile.c_str());    
        }
        else
        {
            ret = -1;
            return ret;
        }
    }

    AVCodec *out_vid_codec,*out_aud_codec;
    out_vid_codec = out_aud_codec = NULL;

    if(outfmt->video_codec != AV_CODEC_ID_NONE && m_in_vid_strm != NULL)
    {
        out_vid_codec = avcodec_find_encoder(outfmt->video_codec);
        if(NULL == out_vid_codec)
        {
            PRINT_MSG("Could Not Find Vid Encoder")
            ret = -1;
            return ret;
        }
        else
        {
            PRINT_MSG("Found Out Vid Encoder ")
            m_out_vid_strm = avformat_new_stream(m_outformat, out_vid_codec);
            if(NULL == m_out_vid_strm)
            {
                 PRINT_MSG("Failed to Allocate Output Vid Strm ")
                 ret = -1;
                 return ret;
            }
            else
            {
                 PRINT_MSG("Allocated Video Stream ")
                 if(avcodec_copy_context(m_out_vid_strm->codec, m_informat->streams[m_in_vid_strm_idx]->codec) != 0)
                 {
                    PRINT_MSG("Failed to Copy Context ")
                    ret = -1;
                    return ret;
                 }
                 else
                 {
                    m_out_vid_strm->sample_aspect_ratio.den = m_out_vid_strm->codec->sample_aspect_ratio.den;
                    m_out_vid_strm->sample_aspect_ratio.num = m_in_vid_strm->codec->sample_aspect_ratio.num;
                    PRINT_MSG("Copied Context ")
                    m_out_vid_strm->codec->codec_id = m_in_vid_strm->codec->codec_id;
                    m_out_vid_strm->codec->time_base.num = 1;
                    m_out_vid_strm->codec->time_base.den = m_fps*(m_in_vid_strm->codec->ticks_per_frame);         
                    m_out_vid_strm->time_base.num = 1;
                    m_out_vid_strm->time_base.den = 1000;
                    m_out_vid_strm->r_frame_rate.num = m_fps;
                    m_out_vid_strm->r_frame_rate.den = 1;
                    m_out_vid_strm->avg_frame_rate.den = 1;
                    m_out_vid_strm->avg_frame_rate.num = m_fps;
                    m_out_vid_strm->duration = (m_out_end_time - m_out_start_time)*1000;
                 }
               }
            }
      }

    if(outfmt->audio_codec != AV_CODEC_ID_NONE && m_in_aud_strm != NULL)
    {
        out_aud_codec = avcodec_find_encoder(outfmt->audio_codec);
        if(NULL == out_aud_codec)
        {
            PRINT_MSG("Could Not Find Out Aud Encoder ")
            ret = -1;
            return ret;
        }
        else
        {
            PRINT_MSG("Found Out Aud Encoder ")
            m_out_aud_strm = avformat_new_stream(m_outformat, out_aud_codec);
            if(NULL == m_out_aud_strm)
            {
                PRINT_MSG("Failed to Allocate Out Vid Strm ")
                ret = -1;
                return ret;
            }
            else
            {
                if(avcodec_copy_context(m_out_aud_strm->codec, m_informat->streams[m_in_aud_strm_idx]->codec) != 0)
                {
                    PRINT_MSG("Failed to Copy Context ")
                    ret = -1;
                    return ret;
                }
                else
                 {
                    PRINT_MSG("Copied Context ")
                    m_out_aud_strm->codec->codec_id = m_in_aud_strm->codec->codec_id;
                    m_out_aud_strm->codec->codec_tag = 0;
                    m_out_aud_strm->pts = m_in_aud_strm->pts;
                    m_out_aud_strm->duration = m_in_aud_strm->duration;
                    m_out_aud_strm->time_base.num = m_in_aud_strm->time_base.num;
                    m_out_aud_strm->time_base.den = m_in_aud_strm->time_base.den;

                }
            }
         }
      }

      if (!(outfmt->flags & AVFMT_NOFILE)) 
      {
        if (avio_open2(&m_outformat->pb, outfile.c_str(), AVIO_FLAG_WRITE,NULL, NULL) < 0) 
        {
                PRINT_VAL("Could Not Open File ", outfile)
                ret = -1;
                return ret;
        }
      }
        /* Write the stream header, if any. */
      if (avformat_write_header(m_outformat, NULL) < 0) 
      {
            PRINT_VAL("Error Occurred While Writing Header ", outfile)
            ret = -1;
            return ret;
      }
      else
      {
            PRINT_MSG("Written Output header ")
            m_init_done = true;
      }

    return ret;
}

int VideoClipper::GenerateClip(void)
{
    AVPacket pkt, outpkt;
    int aud_pts = 0, vid_pts = 0, aud_dts = 0, vid_dts = 0;
    int last_vid_pts = 0;
    if(m_good_clip)
    {
        SeekFrame();
        while(av_read_frame(m_informat, &pkt) >= 0 && (m_num_frames-- > 0))
        {
            if(pkt.stream_index == m_in_vid_strm_idx)
            {
                PRINT_VAL("ACTUAL VID Pkt PTS ",av_rescale_q(pkt.pts,m_in_vid_strm->time_base, m_in_vid_strm->codec->time_base))
                PRINT_VAL("ACTUAL VID Pkt DTS ", av_rescale_q(pkt.dts, m_in_vid_strm->time_base, m_in_vid_strm->codec->time_base ))
                av_init_packet(&outpkt);
                if(pkt.pts != AV_NOPTS_VALUE)
                {
                    if(last_vid_pts == vid_pts)
                    {
                        vid_pts++;
                        last_vid_pts = vid_pts;
                    }
                    outpkt.pts = vid_pts;   
                    PRINT_VAL("ReScaled VID Pts ", outpkt.pts)
                }
                else
                {
                    outpkt.pts = AV_NOPTS_VALUE;
                }

                if(pkt.dts == AV_NOPTS_VALUE)
                {
                    outpkt.dts = AV_NOPTS_VALUE;
                }
                else
                {
                    outpkt.dts = vid_pts;
                    PRINT_VAL("ReScaled VID Dts ", outpkt.dts)
                    PRINT_MSG("=======================================")
                }

                outpkt.data = pkt.data;
                outpkt.size = pkt.size;
                outpkt.stream_index = pkt.stream_index;
                outpkt.flags |= AV_PKT_FLAG_KEY;
                last_vid_pts = vid_pts;
                if(av_interleaved_write_frame(m_outformat, &outpkt) < 0)
                {
                    PRINT_MSG("Failed Video Write ")
                }
                else
                {
                    m_out_vid_strm->codec->frame_number++;
                }
                av_free_packet(&outpkt);
                av_free_packet(&pkt);
            }
            else if(pkt.stream_index == m_in_aud_strm_idx)
            {
                PRINT_VAL("ACTUAL AUD Pkt PTS ", av_rescale_q(pkt.pts, m_in_aud_strm->time_base, m_in_aud_strm->codec->time_base))
                PRINT_VAL("ACTUAL AUD Pkt DTS ", av_rescale_q(pkt.dts, m_in_aud_strm->time_base, m_in_aud_strm->codec->time_base))
                //num_aud_pkt++;
                av_init_packet(&outpkt);
                if(pkt.pts != AV_NOPTS_VALUE)
                {
                    outpkt.pts = aud_pts;
                    PRINT_VAL("ReScaled AUD PTS ", outpkt.pts)
                }
                else
                {
                    outpkt.pts = AV_NOPTS_VALUE;
                }

                if(pkt.dts == AV_NOPTS_VALUE)
                {
                    outpkt.dts = AV_NOPTS_VALUE;
                }
                else
                {
                    outpkt.dts = aud_pts;
                    PRINT_VAL("ReScaled AUD DTS ", outpkt.dts)
                    PRINT_MSG("====================================")
                    if( outpkt.pts >= outpkt.dts)
                    {
                        outpkt.dts = outpkt.pts;
                    }
                    if(outpkt.dts == aud_dts)
                    {
                        outpkt.dts++;
                    }
                    if(outpkt.pts < outpkt.dts)
                    {
                        outpkt.pts = outpkt.dts;
                        aud_pts = outpkt.pts;
                    }
                }

                outpkt.data = pkt.data;
                outpkt.size = pkt.size;
                outpkt.stream_index = pkt.stream_index;
                outpkt.flags |= AV_PKT_FLAG_KEY;
                vid_pts = aud_pts;
                aud_pts++;
                if(av_interleaved_write_frame(m_outformat, &outpkt) < 0)
                {
                    PRINT_MSG("Faile Audio Write ")
                }
                else
                {
                    m_out_aud_strm->codec->frame_number++;
                }
                av_free_packet(&outpkt);
                av_free_packet(&pkt);
        }
        else
        {
            PRINT_MSG("Got Unknown Pkt ")
            //num_unkwn_pkt++;
        }
        //num_total_pkt++;
    }

    av_write_trailer(m_outformat);
    av_free_packet(&outpkt);
    av_free_packet(&pkt);
    return 0;    
 }
    return -1;
}

答案 1 :(得分:1)

您可以使用AVBlocks库(example)。我使用示例代码来解决同样的问题。

为了正确的工作,不仅使用.aac我在StreamType :: *和StreamSubType :: *

中做了一些更改