logo标签列表

实时捕获Metal MTKView作为电影？

iosswift3metalavcapture
20
什么是从MTKView捕获帧的最有效方法？如果可能的话，我想实时保存帧到.mov文件中。是否可以将其渲染到AVPlayer框架或类似的东西中？
目前使用以下代码进行绘制（基于@warrenm PerformanceShaders project）：
func draw(in view: MTKView) {
     _ = inflightSemaphore.wait(timeout: DispatchTime.distantFuture)
             updateBuffers()

    let commandBuffer = commandQueue.makeCommandBuffer()

    commandBuffer.addCompletedHandler{ [weak self] commandBuffer in
        if let strongSelf = self {
            strongSelf.inflightSemaphore.signal()
        }
    }
   // Dispatch the current kernel to perform the selected image filter
    selectedKernel.encode(commandBuffer: commandBuffer,
        sourceTexture: kernelSourceTexture!,
        destinationTexture: kernelDestTexture!)

    if let renderPassDescriptor = view.currentRenderPassDescriptor, let currentDrawable = view.currentDrawable
    {
        let clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: 1)
        renderPassDescriptor.colorAttachments[0].clearColor = clearColor

        let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor)
        renderEncoder.label = "Main pass"

        renderEncoder.pushDebugGroup("Draw textured square")
        renderEncoder.setFrontFacing(.counterClockwise)
        renderEncoder.setCullMode(.back)

        renderEncoder.setRenderPipelineState(pipelineState)
        renderEncoder.setVertexBuffer(vertexBuffer, offset: MBEVertexDataSize * bufferIndex, at: 0)
        renderEncoder.setVertexBuffer(uniformBuffer, offset: MBEUniformDataSize * bufferIndex , at: 1)
        renderEncoder.setFragmentTexture(kernelDestTexture, at: 0)
        renderEncoder.setFragmentSamplerState(sampler, at: 0)
        renderEncoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)

        renderEncoder.popDebugGroup()
        renderEncoder.endEncoding()

        commandBuffer.present(currentDrawable)
    }

    bufferIndex = (bufferIndex + 1) % MBEMaxInflightBuffers

    commandBuffer.commit()
 }
这是完全可能的。由于在iOS上从IOSurface创建CVPixelBuffer不是受支持的操作，因此最快的方法可能是使用AVAssetWriterInputPixelBufferAdaptor（它包装了一个像素缓冲池），每帧可以读取可绘制纹理的内容。有一些关于如何使用OpenGL ES的示例。我将尝试发布更完整的答案，其中包含所有具体信息，但也许这会帮助您。另一个要考虑的选择是ReplayKit，如果它足够灵活以满足您的需求。 - warrenm
2个回答
43

这是一个小型类，执行将Metal视图的内容录制到电影文件中的基本功能：

class MetalVideoRecorder {
    var isRecording = false
    var recordingStartTime = TimeInterval(0)

    private var assetWriter: AVAssetWriter
    private var assetWriterVideoInput: AVAssetWriterInput
    private var assetWriterPixelBufferInput: AVAssetWriterInputPixelBufferAdaptor

    init?(outputURL url: URL, size: CGSize) {
        do {
            assetWriter = try AVAssetWriter(outputURL: url, fileType: .m4v)
        } catch {
            return nil
        }

        let outputSettings: [String: Any] = [ AVVideoCodecKey : AVVideoCodecType.h264,
            AVVideoWidthKey : size.width,
            AVVideoHeightKey : size.height ]

        assetWriterVideoInput = AVAssetWriterInput(mediaType: .video, outputSettings: outputSettings)
        assetWriterVideoInput.expectsMediaDataInRealTime = true

        let sourcePixelBufferAttributes: [String: Any] = [
            kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_32BGRA,
            kCVPixelBufferWidthKey as String : size.width,
            kCVPixelBufferHeightKey as String : size.height ]

        assetWriterPixelBufferInput = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: assetWriterVideoInput,
                                                                           sourcePixelBufferAttributes: sourcePixelBufferAttributes)

        assetWriter.add(assetWriterVideoInput)
    }

    func startRecording() {
        assetWriter.startWriting()
        assetWriter.startSession(atSourceTime: .zero)

        recordingStartTime = CACurrentMediaTime()
        isRecording = true
    }

    func endRecording(_ completionHandler: @escaping () -> ()) {
        isRecording = false

        assetWriterVideoInput.markAsFinished()
        assetWriter.finishWriting(completionHandler: completionHandler)
    }

    func writeFrame(forTexture texture: MTLTexture) {
        if !isRecording {
            return
        }

        while !assetWriterVideoInput.isReadyForMoreMediaData {}

        guard let pixelBufferPool = assetWriterPixelBufferInput.pixelBufferPool else {
            print("Pixel buffer asset writer input did not have a pixel buffer pool available; cannot retrieve frame")
            return
        }

        var maybePixelBuffer: CVPixelBuffer? = nil
        let status  = CVPixelBufferPoolCreatePixelBuffer(nil, pixelBufferPool, &maybePixelBuffer)
        if status != kCVReturnSuccess {
            print("Could not get pixel buffer from asset writer input; dropping frame...")
            return
        }

        guard let pixelBuffer = maybePixelBuffer else { return }

        CVPixelBufferLockBaseAddress(pixelBuffer, [])
        let pixelBufferBytes = CVPixelBufferGetBaseAddress(pixelBuffer)!

        // Use the bytes per row value from the pixel buffer since its stride may be rounded up to be 16-byte aligned
        let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
        let region = MTLRegionMake2D(0, 0, texture.width, texture.height)

        texture.getBytes(pixelBufferBytes, bytesPerRow: bytesPerRow, from: region, mipmapLevel: 0)

        let frameTime = CACurrentMediaTime() - recordingStartTime
        let presentationTime = CMTimeMakeWithSeconds(frameTime, preferredTimescale: 240)
        assetWriterPixelBufferInput.append(pixelBuffer, withPresentationTime: presentationTime)

        CVPixelBufferUnlockBaseAddress(pixelBuffer, [])
    }
}

初始化其中之一并调用 startRecording() 后，您可以向包含渲染命令的命令缓冲区添加计划处理程序，并在结束编码但在呈现可绘制项或提交缓冲区之前调用 writeFrame：

let texture = currentDrawable.texture
commandBuffer.addCompletedHandler { commandBuffer in
    self.recorder.writeFrame(forTexture: texture)
}

录制完成后，只需调用endRecording，视频文件将被完成并关闭。

注意事项：

此类假定源纹理为默认格式.bgra8Unorm。如果不是，则会出现崩溃或损坏。如果必要，请使用计算或片段着色器转换纹理，或使用加速库。

此类还假定纹理与视频帧的大小相同。如果不是这种情况（如果可绘制的大小发生变化，或者您的屏幕自动旋转），输出将会损坏，并且可能会看到崩溃。根据应用程序的要求缩放或裁剪源纹理，以减轻此问题。

3
我认为使用定时处理程序是不正确的，因为该块将在GPU执行命令缓冲区之前运行。您需要使用完成处理程序。 - ldoogy
2
还应该注意MTKView的属性framebufferOnly应该设置为false。最近我遇到了这个问题，由于这个问题在视频中出现了伪影/崩溃。 - hartw
刚刚对这个答案进行了编辑，基本上我们需要手动释放像素缓冲区的内存，否则在几秒钟内应用程序将使用所有可用的设备内存并崩溃。 - ImShrey
@zoe，你能解释一下你的修改吗？ - ImShrey
@shreyashshah 对于试图回复或回答的编辑，有一个完整的拒绝理由，并且在第一次审核中使用：“此编辑旨在回复帖子的作者，作为编辑毫无意义。它应该被写成评论或答案。” 它本来就不应该在第一次通过审核。 - Zoe stands with Ukraine
显示剩余8条评论
7

已升级至Swift 5



import AVFoundation

class MetalVideoRecorder {
    var isRecording = false
    var recordingStartTime = TimeInterval(0)

    private var assetWriter: AVAssetWriter
    private var assetWriterVideoInput: AVAssetWriterInput
    private var assetWriterPixelBufferInput: AVAssetWriterInputPixelBufferAdaptor

    init?(outputURL url: URL, size: CGSize) {
        do {
          assetWriter = try AVAssetWriter(outputURL: url, fileType: AVFileType.m4v)
        } catch {
            return nil
        }

      let outputSettings: [String: Any] = [ AVVideoCodecKey : AVVideoCodecType.h264,
            AVVideoWidthKey : size.width,
            AVVideoHeightKey : size.height ]

      assetWriterVideoInput = AVAssetWriterInput(mediaType: AVMediaType.video, outputSettings: outputSettings)
        assetWriterVideoInput.expectsMediaDataInRealTime = true

        let sourcePixelBufferAttributes: [String: Any] = [
            kCVPixelBufferPixelFormatTypeKey as String : kCVPixelFormatType_32BGRA,
            kCVPixelBufferWidthKey as String : size.width,
            kCVPixelBufferHeightKey as String : size.height ]

        assetWriterPixelBufferInput = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: assetWriterVideoInput,
                                                                           sourcePixelBufferAttributes: sourcePixelBufferAttributes)

        assetWriter.add(assetWriterVideoInput)
    }

    func startRecording() {
        assetWriter.startWriting()
      assetWriter.startSession(atSourceTime: CMTime.zero)

        recordingStartTime = CACurrentMediaTime()
        isRecording = true
    }

    func endRecording(_ completionHandler: @escaping () -> ()) {
        isRecording = false

        assetWriterVideoInput.markAsFinished()
        assetWriter.finishWriting(completionHandler: completionHandler)
    }

    func writeFrame(forTexture texture: MTLTexture) {
        if !isRecording {
            return
        }

        while !assetWriterVideoInput.isReadyForMoreMediaData {}

        guard let pixelBufferPool = assetWriterPixelBufferInput.pixelBufferPool else {
            print("Pixel buffer asset writer input did not have a pixel buffer pool available; cannot retrieve frame")
            return
        }

        var maybePixelBuffer: CVPixelBuffer? = nil
        let status  = CVPixelBufferPoolCreatePixelBuffer(nil, pixelBufferPool, &maybePixelBuffer)
        if status != kCVReturnSuccess {
            print("Could not get pixel buffer from asset writer input; dropping frame...")
            return
        }

        guard let pixelBuffer = maybePixelBuffer else { return }

        CVPixelBufferLockBaseAddress(pixelBuffer, [])
        let pixelBufferBytes = CVPixelBufferGetBaseAddress(pixelBuffer)!

        // Use the bytes per row value from the pixel buffer since its stride may be rounded up to be 16-byte aligned
        let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
        let region = MTLRegionMake2D(0, 0, texture.width, texture.height)

        texture.getBytes(pixelBufferBytes, bytesPerRow: bytesPerRow, from: region, mipmapLevel: 0)

        let frameTime = CACurrentMediaTime() - recordingStartTime
        let presentationTime = CMTimeMakeWithSeconds(frameTime, preferredTimescale:   240)
        assetWriterPixelBufferInput.append(pixelBuffer, withPresentationTime: presentationTime)

        CVPixelBufferUnlockBaseAddress(pixelBuffer, [])
    
        // You need to release memory allocated to pixelBuffer
        CVPixelBufferRelease(pixelBuffer)
    }
}

编辑：添加了CVPixelBufferRelease(pixelBuffer)以避免内存泄漏。 没有这个，每帧新创建的像素缓冲区将会一直驻留在内存中，最终应用程序将占用所有可用系统内存。

Xcode给我一个错误提示：'CVPixelBufferRelease'不可用：Core Foundation对象自动进行内存管理。 - Shuai
网页内容由stack overflow 提供, 点击上面的
可以查看英文原文，
原文链接