Question

我正在开发一款iOS应用，以 aac audio 显示 h264 视频流。
我拥有的流是一个不使用 HLS 或 rtsp / rtmp 的自定义流，因此我有自己的代码来处理数据接收。
我收到的数据分为两部分：标题数据和帧数据（用于音频和视频）。我想支持iOS6 +，但如果有必要，我会很擅长。

我最初的想法是将我的帧数据从字节数组转换为UIImage，然后用新帧连续更新UIImageView。这个问题是帧仍然需要先解码。

我查看了 ffmpeg ，但我看到的所有示例都需要一个不适合我的URL或本地文件。我读到使用ffmpeg时可能存在一些许可问题。

我也看了 openh264 。我认为这可能是一个选择，但由于我正在为iOS开发，我仍然会遇到这些许可问题。

修改
我设法使用 videoToolbox 和provided sample在iOS 8+上实现了这一功能。我的问题是我从流中收到的数据多于示例中的数据。

我仍在寻找在iOS 6和7上执行此操作的方法。

所以我的问题是如何处理我的帧的解码和显示？

Answer 1

我最终使用FFmpeg并且没有使用GPL许可证。

这就是我设置它的方式：

我从source forge下载了FFmpeg iOS库。（您也可以从以下位置通过下载构建脚本来构建它：https://github.com/kewlbear/FFmpeg-iOS-build-script）

在代码中，我添加了一个检查，以查看我所使用的操作系统版本：

uint8_t *data = (unsigned char*)buf;
float version = [[[UIDevice currentDevice] systemVersion] floatValue];
if (version >= 8.0)
{
    [self receivedRawVideoFrame:data withSize:ret ];
}
else if (version >= 6.0 && version < 8.0)
{
    [self altDecodeFrame:data withSize:ret isConfigured:configured];
}

您可以看到找到VideoToolbox部件here的实现。

- (void)altDecodeFrame:(uint8_t *)frame_bytes withSize:(int) frameSize isConfigured:(Boolean) configured
{
    if (!configured) {
    uint8_t *header = NULL;

    // I know what my H.264 data source's NALUs look like so I know start code index is always 0.
    // if you don't know where it starts, you can use a for loop similar to how i find the 2nd and 3rd start codes
    int startCodeIndex = 0;
    int secondStartCodeIndex = 0;
    int thirdStartCodeIndex = 0;
    int fourthStartCodeIndex = 0;

    int nalu_type = (frame_bytes[startCodeIndex + 4] & 0x1F);

    // NALU type 7 is the SPS parameter NALU
    if (nalu_type == 7)
    {
        // find where the second PPS start code begins, (the 0x00 00 00 01 code)
        // from which we also get the length of the first SPS code
        for (int i = startCodeIndex + 4; i < startCodeIndex + 40; i++)
        {
            if (frame_bytes[i] == 0x00 && frame_bytes[i+1] == 0x00 && frame_bytes[i+2] == 0x00 && frame_bytes[i+3] == 0x01)
            {
                secondStartCodeIndex = i;
                _spsSize = secondStartCodeIndex;   // includes the header in the size
                break;
            }
        }

        // find what the second NALU type is
        nalu_type = (frame_bytes[secondStartCodeIndex + 4] & 0x1F);
    }

    // type 8 is the PPS parameter NALU
    if(nalu_type == 8)
    {
        // find where the NALU after this one starts so we know how long the PPS parameter is
        for (int i = _spsSize + 4; i < _spsSize + 30; i++)
        {
            if (frame_bytes[i] == 0x00 && frame_bytes[i+1] == 0x00 && frame_bytes[i+2] == 0x00 && frame_bytes[i+3] == 0x01)
            {
                thirdStartCodeIndex = i;
                _ppsSize = thirdStartCodeIndex - _spsSize;
                break;
            }
        }

        // allocate enough data to fit the SPS and PPS parameters into our data object.
        header = malloc(_ppsSize + _spsSize);

        // copy in the actual sps and pps values, again ignoring the 4 byte header
        memcpy (header, &frame_bytes[0], _ppsSize + _spsSize);
        NSLog(@"refresh codec context");
        avcodec_close(instance.codec_context);
        int result;
        // I know I have an H264 stream, so that is the codex I look for
        AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264);
        self.codec_context = avcodec_alloc_context3(codec);
        //open codec
        result = avcodec_open2(self.codec_context, codec,NULL);
        if (result < 0) {
            NSLog(@"avcodec_open2 returned %i", result);
        }

        if (header != NULL) {
            //set the extra data for decoding                
            self.codec_context->extradata = header;
            self.codec_context->extradata_size = _spsSize+_ppsSize;
            self.codec_context->flags      |= CODEC_FLAG_GLOBAL_HEADER;
            free(header);
        }
        // allocate the picture data.
        // My frame data is in PIX_FMT_YUV420P format, but I will be converting that later on.
        avpicture_alloc(&_pictureData, PIX_FMT_RGB24, 1280, 720);

        // After my SPS and PPS data I receive a SEI NALU
        nalu_type = (frame_bytes[thirdStartCodeIndex + 4] & 0x1F);
    }

    if(nalu_type == 6)
    {
        for (int i = _spsSize +_ppsSize + 4; i < _spsSize +_ppsSize + 30; i++)
        {
            if (frame_bytes[i] == 0x00 && frame_bytes[i+1] == 0x00 && frame_bytes[i+2] == 0x00 && frame_bytes[i+3] == 0x01)
            {
                fourthStartCodeIndex = i;
                _seiSize = fourthStartCodeIndex - (_spsSize + _ppsSize);
                break;
            }
        }
        // do stuff here
        // [...]
        nalu_type = (frame_bytes[fourthStartCodeIndex + 4] & 0x1F);
    }
    }

    //I had some issues with a large build up of memory, so I created an autoreleasepool  
    @autoreleasepool {
        _frm = av_frame_alloc();

        int result;
        //fill the packet with the frame data
        av_init_packet(&_pkt);
        _pkt.data = frame_bytes;
        _pkt.size = frameSize;
        _pkt.flags = AV_PKT_FLAG_KEY;

        int got_packet;
        //Decode the frames
        result = avcodec_decode_video2(self.codec_context, _frm, &got_packet, &_pkt);
        if (result < 0) {
            NSLog(@"avcodec_decode_video2 returned %i", result);
        }

        if (_frm == NULL) {
            return;
        }
        else
        {
            //Here we will convert from YUV420P to RGB24
            static int sws_flags =  SWS_FAST_BILINEAR;
            struct SwsContext *img_convert_ctx = sws_getContext(self.codec_context->width, self.codec_context->height, self.codec_context->pix_fmt, 1280, 720, PIX_FMT_RGB24, sws_flags, NULL, NULL, NULL);

            sws_scale(img_convert_ctx, (const uint8_t* const*)_frm->data, _frm->linesize, 0, _frm->height, _pictureData.data, _pictureData.linesize);
            sws_freeContext(img_convert_ctx);

            self.lastImage = [self imageFromAVPicture:_pictureData width:_frm->width height:_frm->height];

            av_frame_unref(_frm);
        }

        if (!self.lastImage) {
            return;
        }

        //Normally we render on the AVSampleBufferDisplayLayer, so hide that.
        //Add a UIImageView and display the image there.
        dispatch_sync(dispatch_get_main_queue(), ^{
            if (![[[self viewController] avSbdLayer] isHidden]) {

                [[[self viewController] avSbdLayer] setHidden:true];
                self.imageView = [[UIImageView alloc] initWithFrame:[[[self viewController] view] bounds]] ;
                [[[self viewController] view] addSubview: self.imageView];

            }

            [[self imageView] setImage: self.lastImage];
        });

        // Free the allocated data
        av_free_packet(&_pkt);
        av_frame_free(&_frm);
        av_free(_frm);
        //    free(bckgrnd);
    }
}

这就是我从AVPicture制作UIImage的方式

-(UIImage *)imageFromAVPicture:(AVPicture)pict width:(int)width height:(int)height {

CGBitmapInfo bitmapInfo = kCGBitmapByteOrderDefault;
CFDataRef data = CFDataCreateWithBytesNoCopy(kCFAllocatorDefault, pict.data[0], pict.linesize[0]*height,kCFAllocatorNull);
CGDataProviderRef provider = CGDataProviderCreateWithCFData(data);
CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
CGImageRef cgImage = CGImageCreate(width,
                                   height,
                                   8,
                                   24,
                                   pict.linesize[0],
                                   colorSpace,
                                   bitmapInfo,
                                   provider,
                                   NULL,
                                   NO,
                                   kCGRenderingIntentDefault);
CGColorSpaceRelease(colorSpace);
UIImage *image = [UIImage imageWithCGImage:cgImage];
CGImageRelease(cgImage);
CGDataProviderRelease(provider);
CFRelease(data);

return image;
}

如果有人有其他（或更好）的解决方案，请告诉我。

如何在iOS 6+上解码h264字节流？

1 个答案: