我使用MTLTexture从CVImageBuffer（来自相机和玩家）创建CVMetalTextureCacheCreateTextureFromImage以获得CVMetalTexture，然后CVMetalTextureGetTexture来得到MTLTexture。
我看到的问题是，当我稍后使用Metal渲染纹理时，我偶尔会看到无序渲染的视频帧（视觉上它会及时地来回断断续续），可能是因为CoreVideo正在修改底层{ {1}}存储，而CVImageBuffer只是指向那里。
有没有办法让CoreVideo不触及那个缓冲区并在其池中使用另一个缓冲区，直到我释放MTLTexture对象？
我当前的解决方法是使用MTLTexture对纹理进行blitting，但是因为我只需要保持纹理约30毫秒，这似乎是不必要的。
答案 0 :(得分：5)
我最近遇到了同样的问题。问题是MTLTexture无效，除非它拥有CVMetalTextureRef仍然存在。您必须在使用MTLTexture的整个过程中保持对CVMetalTextureRef的引用（一直到当前渲染周期结束）。
答案 1 :(得分：2)
我遇到了同样的问题，但是在我的情况下，对CVMetalTexture对象的额外引用并没有解决这个问题。
据我所知，只有在我的金属代码完成处理前一帧之前我从相机收到一个新帧时才会发生这种情况。
似乎CVMetalTextureCacheCreateTextureFromImage只是在像素缓冲区的顶部创建了一个纹理，相机正在将数据输入其中。因此，异步访问Metal代码会导致一些问题。
我决定创建一个MTLTexture副本（它也是异步但速度很快）。
以下是CVMetalTextureCacheCreateTextureFromImage（）的描述
＆＃34;此函数根据指定的内容创建或返回映射到图像缓冲区的缓存CoreVideo Metal纹理缓冲区，在基于设备的图像缓冲区和MTLTexture对象之间创建实时绑定。＆＃34;，< / p>
答案 2 :(得分：0)
似乎您的问题取决于您如何管理会话以获取原始相机数据。
我认为您可以深入和实时地分析相机会话，以了解此课程的会话当前状态（ MetalCameraSession ）：
import AVFoundation
import Metal
public protocol MetalCameraSessionDelegate {
func metalCameraSession(_ session: MetalCameraSession, didReceiveFrameAsTextures: [MTLTexture], withTimestamp: Double)
func metalCameraSession(_ session: MetalCameraSession, didUpdateState: MetalCameraSessionState, error: MetalCameraSessionError?)
}
public final class MetalCameraSession: NSObject {
public var frameOrientation: AVCaptureVideoOrientation? {
didSet {
guard
let frameOrientation = frameOrientation,
let outputData = outputData,
outputData.connection(withMediaType: AVMediaTypeVideo).isVideoOrientationSupported
else { return }
outputData.connection(withMediaType: AVMediaTypeVideo).videoOrientation = frameOrientation
}
}
public let captureDevicePosition: AVCaptureDevicePosition
public var delegate: MetalCameraSessionDelegate?
public let pixelFormat: MetalCameraPixelFormat
public init(pixelFormat: MetalCameraPixelFormat = .rgb, captureDevicePosition: AVCaptureDevicePosition = .back, delegate: MetalCameraSessionDelegate? = nil) {
self.pixelFormat = pixelFormat
self.captureDevicePosition = captureDevicePosition
self.delegate = delegate
super.init();
NotificationCenter.default.addObserver(self, selector: #selector(captureSessionRuntimeError), name: NSNotification.Name.AVCaptureSessionRuntimeError, object: nil)
}
public func start() {
requestCameraAccess()
captureSessionQueue.async(execute: {
do {
self.captureSession.beginConfiguration()
try self.initializeInputDevice()
try self.initializeOutputData()
self.captureSession.commitConfiguration()
try self.initializeTextureCache()
self.captureSession.startRunning()
self.state = .streaming
}
catch let error as MetalCameraSessionError {
self.handleError(error)
}
catch {
print(error.localizedDescription)
}
})
}
public func stop() {
captureSessionQueue.async(execute: {
self.captureSession.stopRunning()
self.state = .stopped
})
}
fileprivate var state: MetalCameraSessionState = .waiting {
didSet {
guard state != .error else { return }
delegate?.metalCameraSession(self, didUpdateState: state, error: nil)
}
}
fileprivate var captureSession = AVCaptureSession()
internal var captureDevice = MetalCameraCaptureDevice()
fileprivate var captureSessionQueue = DispatchQueue(label: "MetalCameraSessionQueue", attributes: [])
#if arch(i386) || arch(x86_64)
#else
/// Texture cache we will use for converting frame images to textures
internal var textureCache: CVMetalTextureCache?
#endif
fileprivate var metalDevice = MTLCreateSystemDefaultDevice()
internal var inputDevice: AVCaptureDeviceInput? {
didSet {
if let oldValue = oldValue {
captureSession.removeInput(oldValue)
}
captureSession.addInput(inputDevice)
}
}
internal var outputData: AVCaptureVideoDataOutput? {
didSet {
if let oldValue = oldValue {
captureSession.removeOutput(oldValue)
}
captureSession.addOutput(outputData)
}
}
fileprivate func requestCameraAccess() {
captureDevice.requestAccessForMediaType(AVMediaTypeVideo) {
(granted: Bool) -> Void in
guard granted else {
self.handleError(.noHardwareAccess)
return
}
if self.state != .streaming && self.state != .error {
self.state = .ready
}
}
}
fileprivate func handleError(_ error: MetalCameraSessionError) {
if error.isStreamingError() {
state = .error
}
delegate?.metalCameraSession(self, didUpdateState: state, error: error)
}
fileprivate func initializeTextureCache() throws {
#if arch(i386) || arch(x86_64)
throw MetalCameraSessionError.failedToCreateTextureCache
#else
guard
let metalDevice = metalDevice,
CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, metalDevice, nil, &textureCache) == kCVReturnSuccess
else {
throw MetalCameraSessionError.failedToCreateTextureCache
}
#endif
}
fileprivate func initializeInputDevice() throws {
var captureInput: AVCaptureDeviceInput!
guard let inputDevice = captureDevice.device(mediaType: AVMediaTypeVideo, position: captureDevicePosition) else {
throw MetalCameraSessionError.requestedHardwareNotFound
}
do {
captureInput = try AVCaptureDeviceInput(device: inputDevice)
}
catch {
throw MetalCameraSessionError.inputDeviceNotAvailable
}
guard captureSession.canAddInput(captureInput) else {
throw MetalCameraSessionError.failedToAddCaptureInputDevice
}
self.inputDevice = captureInput
}
fileprivate func initializeOutputData() throws {
let outputData = AVCaptureVideoDataOutput()
outputData.videoSettings = [
kCVPixelBufferPixelFormatTypeKey as AnyHashable : Int(pixelFormat.coreVideoType)
]
outputData.alwaysDiscardsLateVideoFrames = true
outputData.setSampleBufferDelegate(self, queue: captureSessionQueue)
guard captureSession.canAddOutput(outputData) else {
throw MetalCameraSessionError.failedToAddCaptureOutput
}
self.outputData = outputData
}
@objc
fileprivate func captureSessionRuntimeError() {
if state == .streaming {
handleError(.captureSessionRuntimeError)
}
}
deinit {
NotificationCenter.default.removeObserver(self)
}
}
extension MetalCameraSession: AVCaptureVideoDataOutputSampleBufferDelegate {
#if arch(i386) || arch(x86_64)
#else
private func texture(sampleBuffer: CMSampleBuffer?, textureCache: CVMetalTextureCache?, planeIndex: Int = 0, pixelFormat: MTLPixelFormat = .bgra8Unorm) throws -> MTLTexture {
guard let sampleBuffer = sampleBuffer else {
throw MetalCameraSessionError.missingSampleBuffer
}
guard let textureCache = textureCache else {
throw MetalCameraSessionError.failedToCreateTextureCache
}
guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
throw MetalCameraSessionError.failedToGetImageBuffer
}
let isPlanar = CVPixelBufferIsPlanar(imageBuffer)
let width = isPlanar ? CVPixelBufferGetWidthOfPlane(imageBuffer, planeIndex) : CVPixelBufferGetWidth(imageBuffer)
let height = isPlanar ? CVPixelBufferGetHeightOfPlane(imageBuffer, planeIndex) : CVPixelBufferGetHeight(imageBuffer)
var imageTexture: CVMetalTexture?
let result = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, imageBuffer, nil, pixelFormat, width, height, planeIndex, &imageTexture)
guard
let unwrappedImageTexture = imageTexture,
let texture = CVMetalTextureGetTexture(unwrappedImageTexture),
result == kCVReturnSuccess
else {
throw MetalCameraSessionError.failedToCreateTextureFromImage
}
return texture
}
private func timestamp(sampleBuffer: CMSampleBuffer?) throws -> Double {
guard let sampleBuffer = sampleBuffer else {
throw MetalCameraSessionError.missingSampleBuffer
}
let time = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
guard time != kCMTimeInvalid else {
throw MetalCameraSessionError.failedToRetrieveTimestamp
}
return (Double)(time.value) / (Double)(time.timescale);
}
@objc public func captureOutput(_ captureOutput: AVCaptureOutput!, didOutputSampleBuffer sampleBuffer: CMSampleBuffer!, from connection: AVCaptureConnection!) {
do {
var textures: [MTLTexture]!
switch pixelFormat {
case .rgb:
let textureRGB = try texture(sampleBuffer: sampleBuffer, textureCache: textureCache)
textures = [textureRGB]
case .yCbCr:
let textureY = try texture(sampleBuffer: sampleBuffer, textureCache: textureCache, planeIndex: 0, pixelFormat: .r8Unorm)
let textureCbCr = try texture(sampleBuffer: sampleBuffer, textureCache: textureCache, planeIndex: 1, pixelFormat: .rg8Unorm)
textures = [textureY, textureCbCr]
}
let timestamp = try self.timestamp(sampleBuffer: sampleBuffer)
delegate?.metalCameraSession(self, didReceiveFrameAsTextures: textures, withTimestamp: timestamp)
}
catch let error as MetalCameraSessionError {
self.handleError(error)
}
catch {
print(error.localizedDescription)
}
}
#endif
}
通过这个类来了解不同的会话类型和发生的错误（ MetalCameraSessionTypes ）：
import AVFoundation
public enum MetalCameraSessionState {
case ready
case streaming
case stopped
case waiting
case error
}
public enum MetalCameraPixelFormat {
case rgb
case yCbCr
var coreVideoType: OSType {
switch self {
case .rgb:
return kCVPixelFormatType_32BGRA
case .yCbCr:
return kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
}
}
}
public enum MetalCameraSessionError: Error {
case noHardwareAccess
case failedToAddCaptureInputDevice
case failedToAddCaptureOutput
case requestedHardwareNotFound
case inputDeviceNotAvailable
case captureSessionRuntimeError
case failedToCreateTextureCache
case missingSampleBuffer
case failedToGetImageBuffer
case failedToCreateTextureFromImage
case failedToRetrieveTimestamp
public func isStreamingError() -> Bool {
switch self {
case .noHardwareAccess, .failedToAddCaptureInputDevice, .failedToAddCaptureOutput, .requestedHardwareNotFound, .inputDeviceNotAvailable, .captureSessionRuntimeError:
return true
default:
return false
}
}
public var localizedDescription: String {
switch self {
case .noHardwareAccess:
return "Failed to get access to the hardware for a given media type."
case .failedToAddCaptureInputDevice:
return "Failed to add a capture input device to the capture session."
case .failedToAddCaptureOutput:
return "Failed to add a capture output data channel to the capture session."
case .requestedHardwareNotFound:
return "Specified hardware is not available on this device."
case .inputDeviceNotAvailable:
return "Capture input device cannot be opened, probably because it is no longer available or because it is in use."
case .captureSessionRuntimeError:
return "AVCaptureSession runtime error."
case .failedToCreateTextureCache:
return "Failed to initialize texture cache."
case .missingSampleBuffer:
return "No sample buffer to convert the image from."
case .failedToGetImageBuffer:
return "Failed to retrieve an image buffer from camera's output sample buffer."
case .failedToCreateTextureFromImage:
return "Failed to convert the frame to a Metal texture."
case .failedToRetrieveTimestamp:
return "Failed to retrieve timestamp from the sample buffer."
}
}
}
然后你可以使用包含实例方法的AVFoundation的{{1}}而不是类的包装器（MetalCameraCaptureDevice）：
AVCaptureDevice然后你可以有一个自定义的viewController类来像这样控制相机（ CameraViewController ）：
import AVFoundation
internal class MetalCameraCaptureDevice {
internal func device(mediaType: String, position: AVCaptureDevicePosition) -> AVCaptureDevice? {
guard let devices = AVCaptureDevice.devices(withMediaType: mediaType) as? [AVCaptureDevice] else { return nil }
if let index = devices.index(where: { $0.position == position }) {
return devices[index]
}
return nil
}
internal func requestAccessForMediaType(_ mediaType: String!, completionHandler handler: ((Bool) -> Void)!) {
AVCaptureDevice.requestAccess(forMediaType: mediaType, completionHandler: handler)
}
}
最后你的课可能就像这个（MTKViewController），你有
import UIKit
import Metal
internal final class CameraViewController: MTKViewController {
var session: MetalCameraSession?
override func viewDidLoad() {
super.viewDidLoad()
session = MetalCameraSession(delegate: self)
}
override func viewWillAppear(_ animated: Bool) {
super.viewWillAppear(animated)
session?.start()
}
override func viewDidDisappear(_ animated: Bool) {
super.viewDidDisappear(animated)
session?.stop()
}
}
// MARK: - MetalCameraSessionDelegate
extension CameraViewController: MetalCameraSessionDelegate {
func metalCameraSession(_ session: MetalCameraSession, didReceiveFrameAsTextures textures: [MTLTexture], withTimestamp timestamp: Double) {
self.texture = textures[0]
}
func metalCameraSession(_ cameraSession: MetalCameraSession, didUpdateState state: MetalCameraSessionState, error: MetalCameraSessionError?) {
if error == .captureSessionRuntimeError {
print(error?.localizedDescription ?? "None")
cameraSession.start()
}
DispatchQueue.main.async {
self.title = "Metal camera: \(state)"
}
print("Session changed state to \(state) with error: \(error?.localizedDescription ?? "None").")
}
}
可以准确地获得缓冲相机所期望的public func draw(in: MTKView)
：
MTLTexture现在您拥有所有来源，但您也可以找到所有navoshta（作者） GitHUB 项目的here完成所有关于代码的评论和描述以及关于此项目的精彩教程here
特别是你可以获得纹理的第二部分（你可以在import UIKit
import Metal
#if arch(i386) || arch(x86_64)
#else
import MetalKit
#endif
open class MTKViewController: UIViewController {
open var texture: MTLTexture?
open func willRenderTexture(_ texture: inout MTLTexture, withCommandBuffer commandBuffer: MTLCommandBuffer, device: MTLDevice) {
}
open func didRenderTexture(_ texture: MTLTexture, withCommandBuffer commandBuffer: MTLCommandBuffer, device: MTLDevice) {
}
override open func loadView() {
super.loadView()
#if arch(i386) || arch(x86_64)
NSLog("Failed creating a default system Metal device, since Metal is not available on iOS Simulator.")
#else
assert(device != nil, "Failed creating a default system Metal device. Please, make sure Metal is available on your hardware.")
#endif
initializeMetalView()
initializeRenderPipelineState()
}
fileprivate func initializeMetalView() {
#if arch(i386) || arch(x86_64)
#else
metalView = MTKView(frame: view.bounds, device: device)
metalView.delegate = self
metalView.framebufferOnly = true
metalView.colorPixelFormat = .bgra8Unorm
metalView.contentScaleFactor = UIScreen.main.scale
metalView.autoresizingMask = [.flexibleWidth, .flexibleHeight]
view.insertSubview(metalView, at: 0)
#endif
}
#if arch(i386) || arch(x86_64)
#else
internal var metalView: MTKView!
#endif
internal var device = MTLCreateSystemDefaultDevice()
internal var renderPipelineState: MTLRenderPipelineState?
fileprivate let semaphore = DispatchSemaphore(value: 1)
fileprivate func initializeRenderPipelineState() {
guard
let device = device,
let library = device.newDefaultLibrary()
else { return }
let pipelineDescriptor = MTLRenderPipelineDescriptor()
pipelineDescriptor.sampleCount = 1
pipelineDescriptor.colorAttachments[0].pixelFormat = .bgra8Unorm
pipelineDescriptor.depthAttachmentPixelFormat = .invalid
pipelineDescriptor.vertexFunction = library.makeFunction(name: "mapTexture")
pipelineDescriptor.fragmentFunction = library.makeFunction(name: "displayTexture")
do {
try renderPipelineState = device.makeRenderPipelineState(descriptor: pipelineDescriptor)
}
catch {
assertionFailure("Failed creating a render state pipeline. Can't render the texture without one.")
return
}
}
}
#if arch(i386) || arch(x86_64)
#else
extension MTKViewController: MTKViewDelegate {
public func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {
NSLog("MTKView drawable size will change to \(size)")
}
public func draw(in: MTKView) {
_ = semaphore.wait(timeout: DispatchTime.distantFuture)
autoreleasepool {
guard
var texture = texture,
let device = device
else {
_ = semaphore.signal()
return
}
let commandBuffer = device.makeCommandQueue().makeCommandBuffer()
willRenderTexture(&texture, withCommandBuffer: commandBuffer, device: device)
render(texture: texture, withCommandBuffer: commandBuffer, device: device)
}
}
private func render(texture: MTLTexture, withCommandBuffer commandBuffer: MTLCommandBuffer, device: MTLDevice) {
guard
let currentRenderPassDescriptor = metalView.currentRenderPassDescriptor,
let currentDrawable = metalView.currentDrawable,
let renderPipelineState = renderPipelineState
else {
semaphore.signal()
return
}
let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: currentRenderPassDescriptor)
encoder.pushDebugGroup("RenderFrame")
encoder.setRenderPipelineState(renderPipelineState)
encoder.setFragmentTexture(texture, at: 0)
encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4, instanceCount: 1)
encoder.popDebugGroup()
encoder.endEncoding()
commandBuffer.addScheduledHandler { [weak self] (buffer) in
guard let unwrappedSelf = self else { return }
unwrappedSelf.didRenderTexture(texture, withCommandBuffer: buffer, device: device)
unwrappedSelf.semaphore.signal()
}
commandBuffer.present(currentDrawable)
commandBuffer.commit()
}
}
#endif
类中找到以下代码）：
MetalCameraSession答案 3 :(得分：0)
由于您的相机输入可能会出现问题。如果您的素材与预期输出的帧速率不完全相同，则帧速率不匹配会导致奇怪的重影。尝试禁用自动调整帧速率。
此问题的其他原因可能是由于以下原因：
关键速度：某些速度与帧速率同步，因此会导致口吃。帧率越低，问题越明显。
SUB PIXEL INTERPOLATION：还有其他情况，帧之间的子像素插值会导致细节区域在帧之间闪烁。
成功渲染的解决方案是为帧速率使用正确的速度（每秒像素数），添加足够的运动模糊以隐藏问题，或减少图像中的细节数量。