从视频帧的CMSampleBuffer转换的cv :: Mat失真

Question

我使用AVAssetReader / AVAssetReaderTrackOutput从视频中获取CMSampleBuffer。但是，当我将CMSampleBuffer转换为cv :: Mat时，Mat是扭曲的图像。

视频解码代码：

@objc open func startReading() -> Void {
    if let reader = try? AVAssetReader.init(asset: _asset){

        let videoTrack = _asset.tracks(withMediaType: .video).compactMap{ $0 }.first;
        let options = [kCVPixelBufferPixelFormatTypeKey : Int(kCVPixelFormatType_32BGRA)]
        let readerOutput = AVAssetReaderTrackOutput.init(track: videoTrack!, outputSettings: options as [String : Any])
        reader.add(readerOutput)
        reader.startReading()

        var count = 0
        //reading
        while (reader.status == .reading && videoTrack?.nominalFrameRate != 0){
            let sampleBuffer = readerOutput.copyNextSampleBuffer()
            _delegate?.reader(self, newFrameReady: sampleBuffer, count)
            count = count+1;
        }
        _delegate?.readerDidFinished(self,totalFrameCount: count)
    }
}

图像秘密代码：

//convert sampleBuffer in callback of video reader
CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
size_t width = CVPixelBufferGetWidth(imageBuffer);
size_t height = CVPixelBufferGetHeight(imageBuffer);
CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
char *baseBuffer = (char*)CVPixelBufferGetBaseAddress(imageBuffer);

cv::Mat cvImage = cv::Mat((int)height,(int)width,CV_8UC3);

cv::MatIterator_<cv::Vec3b> it_start = cvImage.begin<cv::Vec3b>();
cv::MatIterator_<cv::Vec3b> it_end = cvImage.end<cv::Vec3b>();
long cur = 0;
while (it_start != it_end) {
    //opt pixel
    long p_idx = cur*4;
    char b = baseBuffer[p_idx];
    char g = baseBuffer[p_idx + 1];
    char r = baseBuffer[p_idx + 2];
    cv::Vec3b newpixel(b,g,r);
    *it_start = newpixel;
    cur++;
    it_start++;
}

UIImage *tmpImg = MatToUIImage(cvImage);

tmpImg预览：

我发现有些视频效果不错，但有些效果不好。任何帮助表示赞赏！

Answer 1

最后我发现这个错误是因为填充sampleBuffer字节。

许多API在图像行后面填充了多余的字节，以优化SIMD的内存布局，从而可以处理并行像素。

爆破代码有效。

cv::Mat cvImage = cv::Mat((int)height,(int)width,CV_8UC3);
cv::MatIterator_<cv::Vec3b> it_start = cvImage.begin<cv::Vec3b>();
cv::MatIterator_<cv::Vec3b> it_end = cvImage.end<cv::Vec3b>();
long cur = 0;
//Padding bytes added behind image row bytes
size_t padding = CVPixelBufferGetBytesPerRow(imageBuffer) - width*4;
size_t offset = 0;
while (it_start != it_end) {
    //opt pixel
    long p_idx = cur*4 + offset;
    char b = baseBuffer[p_idx];
    char g = baseBuffer[p_idx + 1];
    char r = baseBuffer[p_idx + 2];
    cv::Vec3b newpixel(b,g,r);
    *it_start = newpixel;
    cur++;
    it_start++;
    if (cur%width == 0) {
        offset = offset + padding;
    }
}

UIImage *tmpImg = MatToUIImage(cvImage);