使用ffmpeg将帧编码为视频

Question

我正在尝试使用C ++在虚幻引擎4中编码视频。我可以访问单独的框架。下面是显示viewport's显示像素并存储在缓冲区中的代码。

//Safely get render target resource.
FRenderTarget* RenderTarget = TextureRenderTarget->GameThread_GetRenderTargetResource();
FIntPoint Size = RenderTarget->GetSizeXY();
auto ImageBytes = Size.X* Size.Y * static_cast<int32>(sizeof(FColor));
TArray<uint8> RawData;
RawData.AddUninitialized(ImageBytes);

//Get image raw data.
if (!RenderTarget->ReadPixelsPtr((FColor*)RawData.GetData()))
{
    RawData.Empty();
    UE_LOG(ExportRenderTargetBPFLibrary, Error, TEXT("ExportRenderTargetAsImage: Failed to get raw data."));
    return false;
}

Buffer::getInstance().add(RawData);

虚幻引擎具有IImageWrapperModule，您可以使用avcodec_encode_video从帧中获取图像，但注意视频编码。我想要的是实时编码帧以用于直播服务。

我发现这篇帖子Encoding a screenshot into a video using FFMPEG这就是我想要的，但是我在解决这个问题时遇到了问题。代码已过时（例如avcodec_encode_video2已更改为void Compressor::DoWork() { AVCodec* codec; AVCodecContext* c = NULL; //uint8_t* outbuf; //int /*i, out_size,*/ outbuf_size; UE_LOG(LogTemp, Warning, TEXT("encoding")); codec = avcodec_find_encoder(AV_CODEC_ID_MPEG1VIDEO); // finding the H264 encoder if (!codec) { UE_LOG(LogTemp, Warning, TEXT("codec not found")); exit(1); } else UE_LOG(LogTemp, Warning, TEXT("codec found")); c = avcodec_alloc_context3(codec); c->bit_rate = 400000; c->width = 1280; // resolution must be a multiple of two (1280x720),(1900x1080),(720x480) c->height = 720; c->time_base.num = 1; // framerate numerator c->time_base.den = 25; // framerate denominator c->gop_size = 10; // emit one intra frame every ten frames c->max_b_frames = 1; // maximum number of b-frames between non b-frames c->keyint_min = 1; // minimum GOP size c->i_quant_factor = (float)0.71; // qscale factor between P and I frames //c->b_frame_strategy = 20; ///// find out exactly what this does c->qcompress = (float)0.6; ///// find out exactly what this does c->qmin = 20; // minimum quantizer c->qmax = 51; // maximum quantizer c->max_qdiff = 4; // maximum quantizer difference between frames c->refs = 4; // number of reference frames c->trellis = 1; // trellis RD Quantization c->pix_fmt = AV_PIX_FMT_YUV420P; // universal pixel format for video encoding c->codec_id = AV_CODEC_ID_MPEG1VIDEO; c->codec_type = AVMEDIA_TYPE_VIDEO; if (avcodec_open2(c, codec, NULL) < 0) { UE_LOG(LogTemp, Warning, TEXT("could not open codec")); // opening the codec //exit(1); } else UE_LOG(LogTemp, Warning, TEXT("codec oppened")); FString FinalFilename = FString("C:/Screen/sample.mpg"); auto &PlatformFile = FPlatformFileManager::Get().GetPlatformFile(); auto FileHandle = PlatformFile.OpenWrite(*FinalFilename, true); if (FileHandle) { delete FileHandle; // remove when ready UE_LOG(LogTemp, Warning, TEXT("file opened")); while (true) { UE_LOG(LogTemp, Warning, TEXT("removing from buffer")); int nbytes = avpicture_get_size(AV_PIX_FMT_YUV420P, c->width, c->height); // allocating outbuffer uint8_t* outbuffer = (uint8_t*)av_malloc(nbytes * sizeof(uint8_t)); AVFrame* inpic = av_frame_alloc(); AVFrame* outpic = av_frame_alloc(); outpic->pts = (int64_t)((float)1 * (1000.0 / ((float)(c->time_base.den))) * 90); // setting frame pts avpicture_fill((AVPicture*)inpic, (uint8_t*)Buffer::getInstance().remove().GetData(), AV_PIX_FMT_PAL8, c->width, c->height); // fill image with input screenshot avpicture_fill((AVPicture*)outpic, outbuffer, AV_PIX_FMT_YUV420P, c->width, c->height); // clear output picture for buffer copy av_image_alloc(outpic->data, outpic->linesize, c->width, c->height, c->pix_fmt, 1); /* inpic->data[0] += inpic->linesize[0]*(screenHeight-1); // flipping frame inpic->linesize[0] = -inpic->linesize[0]; // flipping frame struct SwsContext* fooContext = sws_getContext(screenWidth, screenHeight, PIX_FMT_RGB32, c->width, c->height, PIX_FMT_YUV420P, SWS_FAST_BILINEAR, NULL, NULL, NULL); sws_scale(fooContext, inpic->data, inpic->linesize, 0, c->height, outpic->data, outpic->linesize); // converting frame size and format out_size = avcodec_encode_video(c, outbuf, outbuf_size, outpic); // save in file */ } delete FileHandle; } else { UE_LOG(LogTemp, Warning, TEXT("Can't open file")); } } ，但参数不同。

Bellow是编码器的代码。

avcodec_encode_video2

有人可以解释翻转框架部分（为什么要这样做？）以及如何使用avcodec_encode_video函数代替{{1}}？

Answer 1

不仅avcodec_encode_video已过时，avcodec_encode_video2已被标记为已弃用一段时间。您现在应该使用新的avcodec_send_frame和avcodec_receive_packet进行编码。

“翻转”部分对编码没有任何好处，我强烈建议不要在代码中执行此操作。如果您发现输出大小不正确，只需将swscale插值算法标志切换为SWS_ACCURATE_RND。

除了旧的avcodec_encode_video API，还有几个潜在的风险：

• 要使用H264编码器，请使用AV_CODEC_ID_H264，而不是AV_CODEC_ID_MPEG1VIDEO找到它，同时应使用libx264构建ffmpeg库。
  • 或者，如果您有一张支持nvenc的nvidia卡，avcodec_find_encoder_by_name("h264_nvenc")会更好。
• delete FileHandle执行两次。
• avpicture...函数已被弃用了很长时间。请改用其他功能。

如果性能至关重要，请将所有编码过程移至独立线程而不是游戏线程。

我在自定义GameViewportClient课程中有一些用于编码UE4视口输出的代码，类似于ffmpeg官方muxing和encode_video示例。

MyGameViewportClient.h：

UCLASS(Config=Game)
class FUSIONCUT_API UMyGameViewportClient : public UGameViewportClient
{
    GENERATED_BODY()

public:
    virtual void Draw(FViewport* Viewport, FCanvas* SceneCanvas) override;

    void FirstTimeInit();

    void InitCodec();

    void TidyUp();

    void SetAutoRecording(bool val);
    void RecordNextFrame();
    bool CanRecordNextFrame();
    void SetRecording(bool val);
    void SetLevelDelay(int32 delay);

    void SetOver(bool val);
    void SetAbandon(bool val);
    void SetFilePath(FString out_file);
    void SetThumbnail(FString thumbnail_file, int32 thumbnail_frame);
    void SaveThumbnailImage();

private:
    UPROPERTY(Config)
    FString DeviceNum;

    UPROPERTY(Config)
    FString H264Crf;

    UPROPERTY(Config)
    int DeviceIndex;

    UPROPERTY()
    UFunction* ProgressFunc;

    UPROPERTY()
    UFunction* FinishFunc;

    FIntPoint ViewportSize;
    int count;

    TArray<FColor> ColorBuffer;
    TArray<uint8> IMG_Buffer;

    struct OutputStream {
        AVStream* Stream;
        AVCodecContext* Ctx;

        int64_t NextPts;

        AVFrame* Frame;

        struct SwsContext* SwsCtx;
    };

    OutputStream VideoSt = { 0 };
    AVOutputFormat* Fmt;
    AVFormatContext* FmtCtx;
    AVCodec* VideoCodec;
    AVDictionary* Opt = nullptr;
    SwsContext* SwsCtx;
    AVPacket Pkt;

    int GotOutput;
    int InLineSize[1];

    bool Start;
    bool Over;
    bool FirstTime;
    bool Abandon;
    bool AutoRecording;
    bool RecordingNextFrame;
    double LastSendingTime;
    std::string FilePath;
    FString UEFilePath;
    int32 LevelDelay;

    void EncodeAndWrite();

    void CaptureFrame();
    void AddStream(enum AVCodecID CodecID);
    void OpenVideo();
    int WriteFrame(bool need_save_thumbnail = true);
    void CloseStream();
    void AllocPicture();

    int FFmpegEncode(AVFrame *frame);
};

MyGameViewportClient.cpp：

void UMyGameViewportClient::InitCodec()
{
    ViewportSize = Viewport->GetSizeXY();

    av_register_all();
    avformat_alloc_output_context2(&FmtCtx, nullptr, nullptr, FilePath.c_str());
    if (!FmtCtx)
    {
        UE_LOG(LogTemp, Error, TEXT("cannot alloc format context"));
        return;
    }
    Fmt = FmtCtx->oformat;

    //auto codec_id = AV_CODEC_ID_H264;
    const char codec_name[32] = "h264_nvenc";
    //auto codec = avcodec_find_encoder(codec_id);
    auto codec = avcodec_find_encoder_by_name(codec_name);

    av_format_set_video_codec(FmtCtx, codec);

    if (Fmt->video_codec != AV_CODEC_ID_NONE)
    {
        AddStream(Fmt->video_codec);
    }
    OpenVideo();
    VideoSt.NextPts = 0;
    av_dump_format(FmtCtx, 0, FilePath.c_str(), 1);

    if (!(Fmt->flags & AVFMT_NOFILE))
    {
        auto ret = avio_open(&FmtCtx->pb, FilePath.c_str(), AVIO_FLAG_WRITE);
        if (ret < 0)
        {
            auto errstr = FString(av_err2str(ret));
            UE_LOG(LogTemp, Error, TEXT("Could not open %s: %s"), *UEFilePath, *errstr);
            return;
        }
    }

    auto ret = avformat_write_header(FmtCtx, &Opt);
    if (ret < 0)
    {
        UE_LOG(LogTemp, Error, TEXT("Error occurred when writing header to: %s"), *UEFilePath);
        return;
    }

    InLineSize[0] = 4 * VideoSt.Ctx->width;
    SwsCtx = sws_getContext(VideoSt.Ctx->width, VideoSt.Ctx->height, AV_PIX_FMT_RGBA,
                            VideoSt.Ctx->width, VideoSt.Ctx->height, VideoSt.Ctx->pix_fmt,
                            0, nullptr, nullptr, nullptr);
}

void UMyGameViewportClient::OpenVideo()
{
    auto c = VideoSt.Ctx;
    AVDictionary* opt = nullptr;

    av_dict_copy(&opt, Opt, 0);

    auto ret = avcodec_open2(c, VideoCodec, &opt);
    av_dict_free(&opt);
    if (ret < 0)
    {
        auto errstr = FString(av_err2str(ret));
        UE_LOG(LogTemp, Error, TEXT("Could not open video codec: %s"), *errstr);
    }

    AllocPicture();
    if (!VideoSt.Frame)
    {
        UE_LOG(LogTemp, Error, TEXT("Could not allocate video frame"));
        return;
    }
    if (avcodec_parameters_from_context(VideoSt.Stream->codecpar, c))
    {
        UE_LOG(LogTemp, Error, TEXT("Could not copy the stream parameters"));
    }
}

void UMyGameViewportClient::AllocPicture()
{
    VideoSt.Frame = av_frame_alloc();
    if (!VideoSt.Frame)
    {
        UE_LOG(LogTemp, Error, TEXT("av_frame_alloc failed."));
        return;
    }

    VideoSt.Frame->format = VideoSt.Ctx->pix_fmt;
    VideoSt.Frame->width = ViewportSize.X;
    VideoSt.Frame->height = ViewportSize.Y;

    if (av_frame_get_buffer(VideoSt.Frame, 32) < 0)
    {
        UE_LOG(LogTemp, Error, TEXT("Could not allocate frame data"));
    }
}

void UMyGameViewportClient::AddStream(enum AVCodecID CodecID)
{
    VideoCodec = avcodec_find_encoder(CodecID);
    if (!VideoCodec)
    {
        UE_LOG(LogTemp, Error, TEXT("Could not find encoder for '%s'"), ANSI_TO_TCHAR(avcodec_get_name(CodecID)));
    }


    VideoSt.Stream = avformat_new_stream(FmtCtx, nullptr);
    if (!VideoSt.Stream)
    {
        UE_LOG(LogTemp, Error, TEXT("Could not allocate stream"));
    }

    VideoSt.Stream->id = FmtCtx->nb_streams - 1;
    VideoSt.Ctx = avcodec_alloc_context3(VideoCodec);
    if (!VideoSt.Ctx)
    {
        UE_LOG(LogTemp, Error, TEXT("Could not alloc an encoding context"));
    }

    VideoSt.Ctx->codec_id = CodecID;
    VideoSt.Ctx->width = ViewportSize.X;
    VideoSt.Ctx->height = ViewportSize.Y;
    VideoSt.Stream->time_base = VideoSt.Ctx->time_base = { 1, FRAMERATE };
    VideoSt.Ctx->gop_size = 10;
    VideoSt.Ctx->max_b_frames = 1;
    VideoSt.Ctx->pix_fmt = AV_PIX_FMT_YUV420P;

    av_opt_set(VideoSt.Ctx->priv_data, "cq", TCHAR_TO_ANSI(*H264Crf), 0);  // change `cq` to `crf` if using libx264
    av_opt_set(VideoSt.Ctx->priv_data, "gpu", TCHAR_TO_ANSI(*DeviceNum), 0); // comment this line if using libx264

    if (FmtCtx->oformat->flags & AVFMT_GLOBALHEADER)
        VideoSt.Ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}

void UMyGameViewportClient::EncodeAndWrite()
{
    Pkt = { nullptr };
    av_init_packet(&Pkt);

    fflush(stdout);

    IMG_Buffer.SetNum(ColorBuffer.Num() * 4);
    uint8* DestPtr = nullptr;
    for (auto i = 0; i < ColorBuffer.Num(); i++)
    {
        DestPtr = &IMG_Buffer[i * 4];
        auto SrcPtr = ColorBuffer[i];
        *DestPtr++ = SrcPtr.R;
        *DestPtr++ = SrcPtr.G;
        *DestPtr++ = SrcPtr.B;
        *DestPtr++ = SrcPtr.A;
    }

    uint8* inData[1] = { IMG_Buffer.GetData() };
    sws_scale(SwsCtx, inData, InLineSize, 0, VideoSt.Ctx->height, VideoSt.Frame->data, VideoSt.Frame->linesize);

    VideoSt.Frame->pts = VideoSt.NextPts++;
    if (FFmpegEncode(VideoSt.Frame) < 0)
        UE_LOG(LogTemp, Error, TEXT("Error encoding frame %d"), count);

    auto ret = WriteFrame();
    if (ret < 0)
    {
        auto errstr = FString(av_err2str(ret));
        UE_LOG(LogTemp, Error, TEXT("Error while writing video frame: %s"), *errstr);
    }
    av_packet_unref(&Pkt);
}

int UMyGameViewportClient::WriteFrame()
{
    av_packet_rescale_ts(&Pkt, VideoSt.Ctx->time_base, VideoSt.Stream->time_base);
    Pkt.stream_index = VideoSt.Stream->index;
    return av_interleaved_write_frame(FmtCtx, &Pkt);
}

int UMyGameViewportClient::FFmpegEncode(AVFrame *frame) {
    GotOutput = 0;
    auto ret = avcodec_send_frame(VideoSt.Ctx, frame);
    if (ret < 0 && ret != AVERROR_EOF) {
        auto errstr = FString(av_err2str(ret));
        UE_LOG(LogTemp, Warning, TEXT("error during sending frame, error : %s"), *errstr);
        return -1;
    }

    ret = avcodec_receive_packet(VideoSt.Ctx, &Pkt);
    if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
        return 0;

    if (ret < 0)
    {
        auto errstr = FString(av_make_error_string(ret).c_str());
        UE_LOG(LogTemp, Error, TEXT("Error during receiving frame, error : %s"), *errstr);
        av_packet_unref(&Pkt);
        return -1;
    }

    GotOutput = 1;
    return 0;
}

void UMyGameViewportClient::CloseStream()
{
    avcodec_free_context(&VideoSt.Ctx);
    av_frame_free(&VideoSt.Frame);
    sws_freeContext(SwsCtx);

    if (!(Fmt->flags & AVFMT_NOFILE))
    {
        auto ret = avio_closep(&FmtCtx->pb);
        if (ret < 0)
        {
            auto errstr = FString(av_err2str(ret));
            UE_LOG(LogTemp, Error, TEXT("avio close failed: %s"), *errstr);
        }
    }

    avformat_free_context(FmtCtx);
}

void UMyGameViewportClient::TidyUp()
{
    /* get the delayed frames */
    for (GotOutput = 1; GotOutput; count++)
    {
        fflush(stdout);

        FFmpegEncode(nullptr);

        if (GotOutput)
        {
            auto ret = WriteFrame(false);
            if (ret < 0)
            {
                auto errstr = FString(av_err2str(ret));
                UE_LOG(LogTemp, Error, TEXT("Error while writing video frame: %s"), *errstr);
            }
            av_packet_unref(&Pkt);
        }
    }

    auto ret = av_write_trailer(FmtCtx);
    if (ret < 0)
    {
        auto errstr = FString(av_err2str(ret));
        UE_LOG(LogTemp, Error, TEXT("writing trailer error: %s"), *errstr);
    }

    CloseStream();
}

void UMyGameViewportClient::Draw(FViewport* Viewport, FCanvas* SceneCanvas)
{
    Super::Draw(Viewport, SceneCanvas);
    if (Over)  // You may need to set this in other class
    {
        Over = false;
        TidyUp();
    }

    else {
        CaptureFrame();
    }
}

void UMyGameViewportClient::CaptureFrame()
{
    if (!Viewport) {
        UE_LOG(LogTemp, Error, TEXT("No viewport"));
        return;
    }

    if (ViewportSize.X == 0 || ViewportSize.Y == 0) {
        UE_LOG(LogTemp, Error, TEXT("Viewport size is 0"));
        return;
    }

    ColorBuffer.Empty();

    if (!Viewport->ReadPixels(ColorBuffer, FReadSurfaceDataFlags(),
                              FIntRect(0, 0, ViewportSize.X, ViewportSize.Y)))
    {
        UE_LOG(LogTemp, Error, TEXT("Cannot read from viewport"));
        return;
    }

    EncodeAndWrite();  // call InitCodec() before this
}