Question

我正在尝试使用Metal来理解操作视频输出（CMPixelBuffer）的正确方法。

据我所知，有MTKView。视频输出中的每个CMPixelBuffer都被分配给Metal Texture的某些内容。所以最后的预览是来自MTKView？

当我在屏幕上看到最终结果时，是：

1)CMSampleBuffer->Metal->CMSampleBuffer

或

2)CMSampleBuffer->Metal->MTKView

很困惑。有人可以把东西放在现场吗？

Answer 1

我鼓励您分两步完成此操作。首先，使用Metal和MTKView可以轻松地在屏幕上绘制任何内容。其次，在您知道如何使用Metal绘制（创建和编码命令缓冲区，呈现绘图等）之后，您可以将从CMSampleBuffer生成的纹理应用于全屏四边形或您想要的任何其他几何体

@McZonk提供了here的摄像头捕捉样本，它实现了一个简单的例子。它使用双平面YCbCr样本缓冲区，但是如果可用，您还可以请求BGRA格式化的样本缓冲区，然后可以将其转换为带有MTLTexture的单个CVMetalTextureCacheCreateTextureFromImage并在着色器中进行采样，无需进一步手动转换。

Answer 2

从摄像机生成的样本缓冲区转换为Metal纹理所需的最简单的准系统最小代码：

@import UIKit;
@import AVFoundation;
@import CoreMedia;
#import <MetalKit/MetalKit.h>
#import <Metal/Metal.h>
#import <MetalPerformanceShaders/MetalPerformanceShaders.h>

@interface ViewController : UIViewController <MTKViewDelegate, AVCaptureVideoDataOutputSampleBufferDelegate>  {
    NSString *_displayName;
    NSString *serviceType;
}

@property (retain, nonatomic) SessionContainer *session;
@property (retain, nonatomic) AVCaptureSession *avSession;

@end;

#import "ViewController.h"

@interface ViewController () {
    MTKView *_metalView;

    id<MTLDevice> _device;
    id<MTLCommandQueue> _commandQueue;
    id<MTLTexture> _texture;

    CVMetalTextureCacheRef _textureCache;
}

@property (strong, nonatomic) AVCaptureDevice *videoDevice;
@property (nonatomic) dispatch_queue_t sessionQueue;

@end

@implementation ViewController

- (void)viewDidLoad {
    NSLog(@"%s", __PRETTY_FUNCTION__);
    [super viewDidLoad];

    _device = MTLCreateSystemDefaultDevice();
    _metalView = [[MTKView alloc] initWithFrame:self.view.bounds];
    [_metalView setContentMode:UIViewContentModeScaleAspectFit];
    _metalView.device = _device;
    _metalView.delegate = self;
    _metalView.clearColor = MTLClearColorMake(1, 1, 1, 1);
    _metalView.colorPixelFormat = MTLPixelFormatBGRA8Unorm;
    _metalView.framebufferOnly = NO;
    _metalView.autoResizeDrawable = NO;

    CVMetalTextureCacheCreate(NULL, NULL, _device, NULL, &_textureCache);

    [self.view addSubview:_metalView];

    self.sessionQueue = dispatch_queue_create( "session queue", DISPATCH_QUEUE_SERIAL );

    if ([self setupCamera]) {
        [_avSession startRunning];
    }
}

- (BOOL)setupCamera {
    NSLog(@"%s", __PRETTY_FUNCTION__);
    @try {
        NSError * error;

            _avSession = [[AVCaptureSession alloc] init];
            [_avSession beginConfiguration];
            [_avSession setSessionPreset:AVCaptureSessionPreset640x480];

            // get list of devices; connect to front-facing camera
            self.videoDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
            if (self.videoDevice == nil) return FALSE;

            AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:self.videoDevice error:&error];
            [_avSession addInput:input];

            dispatch_queue_t sampleBufferQueue = dispatch_queue_create("CameraMulticaster", DISPATCH_QUEUE_SERIAL);

            AVCaptureVideoDataOutput * dataOutput = [[AVCaptureVideoDataOutput alloc] init];
            [dataOutput setAlwaysDiscardsLateVideoFrames:YES];
            [dataOutput setVideoSettings:@{(id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA)}];
            [dataOutput setSampleBufferDelegate:self queue:sampleBufferQueue];

            [_avSession addOutput:dataOutput];
            [_avSession commitConfiguration]; 
    } @catch (NSException *exception) {
        NSLog(@"%s - %@", __PRETTY_FUNCTION__, exception.description);
        return FALSE;
    } @finally {
        return TRUE;
    }

}

- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection
{
    CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
    {
        size_t width = CVPixelBufferGetWidth(pixelBuffer);
        size_t height = CVPixelBufferGetHeight(pixelBuffer);

        CVMetalTextureRef texture = NULL;
        CVReturn status = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, _textureCache, pixelBuffer, NULL, MTLPixelFormatBGRA8Unorm, width, height, 0, &texture);
        if(status == kCVReturnSuccess)
        {
            _metalView.drawableSize = CGSizeMake(width, height);
            _texture = CVMetalTextureGetTexture(texture);
            _commandQueue = [_device newCommandQueue];
            CFRelease(texture);
        }
    }
}

- (void)drawInMTKView:(MTKView *)view {
    // creating command encoder
    if (_texture) {
        id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
        id<MTLTexture> drawingTexture = view.currentDrawable.texture;

        // set up and encode the filter
        MPSImageGaussianBlur *filter = [[MPSImageGaussianBlur alloc] initWithDevice:_device sigma:5];

        [filter encodeToCommandBuffer:commandBuffer sourceTexture:_texture destinationTexture:drawingTexture];

        // committing the drawing
        [commandBuffer presentDrawable:view.currentDrawable];
        [commandBuffer commit];
        _texture = nil;
    }
}

- (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size {

}

@end

了解AVCaptureSession视频输出的金属使用情况

2 个答案: