在使用ReplayKit进行流媒体传输时，如何从CMSampleBuffer中保存视频

Question

我正在将我的应用程序内容流式传输到我的RTMP服务器，并使用RPBroadcastSampleHandler。

其中一个方法是：

override func processSampleBuffer(_ sampleBuffer: CMSampleBuffer, with sampleBufferType: RPSampleBufferType) {
    switch sampleBufferType {
    case .video:
        streamer.appendSampleBuffer(sampleBuffer, withType: .video)
        captureOutput(sampleBuffer)
    case .audioApp:
        streamer.appendSampleBuffer(sampleBuffer, withType: .audio)
        captureAudioOutput(sampleBuffer)
    case .audioMic:
        ()
    }
}

captureOutput方法是：

self.lastSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);

    // Append the sampleBuffer into videoWriterInput
    if self.isRecordingVideo {
        if self.videoWriterInput!.isReadyForMoreMediaData {
            if self.videoWriter!.status == AVAssetWriterStatus.writing {
                let whetherAppendSampleBuffer = self.videoWriterInput!.append(sampleBuffer)
                print(">>>>>>>>>>>>>The time::: \(self.lastSampleTime.value)/\(self.lastSampleTime.timescale)")
                if whetherAppendSampleBuffer {
                    print("DEBUG::: Append sample buffer successfully")
                } else {
                    print("WARN::: Append sample buffer failed")
                }
            } else {
                print("WARN:::The videoWriter status is not writing")
            }
        } else {
            print("WARN:::Cannot append sample buffer into videoWriterInput")
        }
    }

由于该示例缓冲区包含音频/视频数据，我认为可以使用AVKit在流媒体传输时将其保存到本地。因此，我正在流开始时创建一个资产写入器：

    let fileManager = FileManager.default
    let documentsPath = NSSearchPathForDirectoriesInDomains(.documentDirectory, .userDomainMask, true)[0]
    self.videoOutputFullFileName = documentsPath.stringByAppendingPathComponent(str: "test_capture_video.mp4")

    if self.videoOutputFullFileName == nil {
        print("ERROR:The video output file name is nil")
        return
    }

    self.isRecordingVideo = true
    if fileManager.fileExists(atPath: self.videoOutputFullFileName!) {
        print("WARN:::The file: \(self.videoOutputFullFileName!) exists, will delete the existing file")
        do {
            try fileManager.removeItem(atPath: self.videoOutputFullFileName!)
        } catch let error as NSError {
            print("WARN:::Cannot delete existing file: \(self.videoOutputFullFileName!), error: \(error.debugDescription)")
        }

    } else {
        print("DEBUG:::The file \(self.videoOutputFullFileName!) doesn't exist")
    }

    let screen = UIScreen.main
    let screenBounds = info.size
    let videoCompressionPropertys = [
        AVVideoAverageBitRateKey: screenBounds.width * screenBounds.height * 10.1
    ]

    let videoSettings: [String: Any] = [
        AVVideoCodecKey: AVVideoCodecH264,
        AVVideoWidthKey: screenBounds.width,
        AVVideoHeightKey: screenBounds.height,
        AVVideoCompressionPropertiesKey: videoCompressionPropertys
    ]

    self.videoWriterInput = AVAssetWriterInput(mediaType: AVMediaTypeVideo, outputSettings: videoSettings)

    guard let videoWriterInput = self.videoWriterInput else {
        print("ERROR:::No video writer input")
        return
    }

    videoWriterInput.expectsMediaDataInRealTime = true

    // Add the audio input
    var acl = AudioChannelLayout()
    memset(&acl, 0, MemoryLayout<AudioChannelLayout>.size)
    acl.mChannelLayoutTag = kAudioChannelLayoutTag_Mono;
    let audioOutputSettings: [String: Any] =
        [ AVFormatIDKey: kAudioFormatMPEG4AAC,
          AVSampleRateKey : 44100,
          AVNumberOfChannelsKey : 1,
          AVEncoderBitRateKey : 64000,
          AVChannelLayoutKey : Data(bytes: &acl, count: MemoryLayout<AudioChannelLayout>.size)]

    audioWriterInput = AVAssetWriterInput(mediaType: AVMediaTypeAudio, outputSettings: audioOutputSettings)

    guard let audioWriterInput = self.audioWriterInput else {
        print("ERROR:::No audio writer input")
        return
    }

    audioWriterInput.expectsMediaDataInRealTime = true

    do {
        self.videoWriter = try AVAssetWriter(outputURL: URL(fileURLWithPath: self.videoOutputFullFileName!), fileType: AVFileTypeMPEG4)
    } catch let error as NSError {
        print("ERROR:::::>>>>>>>>>>>>>Cannot init videoWriter, error:\(error.localizedDescription)")
    }

    guard let videoWriter = self.videoWriter else {
        print("ERROR:::No video writer")
        return
    }

    if videoWriter.canAdd(videoWriterInput) {
        videoWriter.add(videoWriterInput)
    } else {
        print("ERROR:::Cannot add videoWriterInput into videoWriter")
    }

    //Add audio input
    if videoWriter.canAdd(audioWriterInput) {
        videoWriter.add(audioWriterInput)
    } else {
        print("ERROR:::Cannot add audioWriterInput into videoWriter")
    }

    if videoWriter.status != AVAssetWriterStatus.writing {
        print("DEBUG::::::::::::::::The videoWriter status is not writing, and will start writing the video.")

        let hasStartedWriting = videoWriter.startWriting()
        if hasStartedWriting {
            videoWriter.startSession(atSourceTime: self.lastSampleTime)
            print("DEBUG:::Have started writting on videoWriter, session at source time: \(self.lastSampleTime)")
            LOG(videoWriter.status.rawValue)
        } else {
            print("WARN:::Fail to start writing on videoWriter")
        }
    } else {
        print("WARN:::The videoWriter.status is writing now, so cannot start writing action on videoWriter")
    }

然后在流的末尾保存并完成写作：

    print("DEBUG::: Starting to process recorder final...")
    print("DEBUG::: videoWriter status: \(self.videoWriter!.status.rawValue)")
    self.isRecordingVideo = false

    guard let videoWriterInput = self.videoWriterInput else {
        print("ERROR:::No video writer input")
        return
    }
    guard let videoWriter = self.videoWriter else {
        print("ERROR:::No video writer")
        return
    }

    guard let audioWriterInput = self.audioWriterInput else {
        print("ERROR:::No audio writer input")
        return
    }

    videoWriterInput.markAsFinished()
    audioWriterInput.markAsFinished()
    videoWriter.finishWriting {
        if videoWriter.status == AVAssetWriterStatus.completed {
            print("DEBUG:::The videoWriter status is completed")

            let fileManager = FileManager.default
            if fileManager.fileExists(atPath: self.videoOutputFullFileName!) {
                print("DEBUG:::The file: \(self.videoOutputFullFileName ?? "") has been saved in documents folder, and is ready to be moved to camera roll")


                let sharedFileURL = fileManager.containerURL(forSecurityApplicationGroupIdentifier: "group.jp.awalker.co.Hotter")
                guard let documentsPath = sharedFileURL?.path else {
                    LOG("ERROR:::No shared file URL path")
                    return
                }
                let finalFilename = documentsPath.stringByAppendingPathComponent(str: "test_capture_video.mp4")

                //Check whether file exists
                if fileManager.fileExists(atPath: finalFilename) {
                    print("WARN:::The file: \(finalFilename) exists, will delete the existing file")
                    do {
                        try fileManager.removeItem(atPath: finalFilename)
                    } catch let error as NSError {
                        print("WARN:::Cannot delete existing file: \(finalFilename), error: \(error.debugDescription)")
                    }
                } else {
                    print("DEBUG:::The file \(self.videoOutputFullFileName!) doesn't exist")
                }

                do {
                    try fileManager.copyItem(at: URL(fileURLWithPath: self.videoOutputFullFileName!), to: URL(fileURLWithPath: finalFilename))
                }
                catch let error as NSError {
                    LOG("ERROR:::\(error.debugDescription)")
                }

                PHPhotoLibrary.shared().performChanges({
                    PHAssetChangeRequest.creationRequestForAssetFromVideo(atFileURL: URL(fileURLWithPath: finalFilename))
                }) { completed, error in
                    if completed {
                        print("Video \(self.videoOutputFullFileName ?? "") has been moved to camera roll")
                    }

                    if error != nil {
                        print ("ERROR:::Cannot move the video \(self.videoOutputFullFileName ?? "") to camera roll, error: \(error!.localizedDescription)")
                    }
                }

            } else {
                print("ERROR:::The file: \(self.videoOutputFullFileName ?? "") doesn't exist, so can't move this file camera roll")
            }
        } else {
            print("WARN:::The videoWriter status is not completed, stauts: \(videoWriter.status)")
        }
    }

我遇到的问题是finishWriting完成代码从未被执行。写入器仍处于“正在写入”状态，因此视频文件未被保存。

如果我删除“finishWriting”行，并只留下完成代码运行，文件将会被保存，但没有正确完成，当我尝试查看它时，它无法播放，因为可能缺少元数据。

有其他的方法吗？我不想使用AVKit来实际开始捕获以保存录制内容，因为它会占用太多CPU，而且RPBroadcastSampleHandler的CMSampleBuffer已经具有视频数据，但也许在这里使用AVKit是错误的选择？

我应该改变什么？如何从CMSampleBuffer中保存视频？

- DmitryoN

---

你需要将屏幕大小乘以屏幕比例，以获得完整的质量。 - Marty

你是怎么解决这个问题的？我也卡在同一行上，如果我删除“finishWriting”这一行，我可以将文件保存在doc.dir中，但我无法播放该文件，也无法将视频文件保存在手机的相机胶卷中。你能帮我吗？ - McDonal_11

@DmitryoN 为什么在 [AVVideoAverageBitRateKey: screenBounds.width * screenBounds.height * 10.1] 中使用了 10.1？ - Martin

感谢你的代码，我找出直接写入组“containerURL”是不行的。首先在文档目录中输出videoWriter，然后完成后，将视频文件复制（或移动）到共享的“containerURL”。我花了数小时才理解错误，你让我度过了美好的一天DmitryoN。 - Martin

@DmitryoN，你能帮我提供完整的代码吗？ - Kushagra Saxena

Answer 1

来自https://developer.apple.com/documentation/avfoundation/avassetwriter/1390432-finishwritingwithcompletionhandl

该方法立即返回并异步执行其工作

当broadcastFinished返回时，您的扩展将被终止。我能让它工作的唯一方法是阻塞该方法，直到视频处理完成。我不确定这是否是正确的方法（似乎很奇怪），但它可以使用。例如：

        var finishedWriting = false
        videoWriter.finishWriting {
            NSLog("DEBUG:::The videoWriter finished writing.")
            if videoWriter.status == .completed {
                NSLog("DEBUG:::The videoWriter status is completed")

                let fileManager = FileManager.default
                if fileManager.fileExists(atPath: self.videoOutputFullFileName!) {
                    NSLog("DEBUG:::The file: \(self.videoOutputFullFileName ?? "") has been saved in documents folder, and is ready to be moved to camera roll")

                    let sharedFileURL = fileManager.containerURL(forSecurityApplicationGroupIdentifier: "group.xxx.com")
                    guard let documentsPath = sharedFileURL?.path else {
                        NSLog("ERROR:::No shared file URL path")
                        finishedWriting = true
                        return
                    }
                    let finalFilename = documentsPath + "/test_capture_video.mp4"

                    //Check whether file exists
                    if fileManager.fileExists(atPath: finalFilename) {
                        NSLog("WARN:::The file: \(finalFilename) exists, will delete the existing file")
                        do {
                            try fileManager.removeItem(atPath: finalFilename)
                        } catch let error as NSError {
                            NSLog("WARN:::Cannot delete existing file: \(finalFilename), error: \(error.debugDescription)")
                        }
                    } else {
                        NSLog("DEBUG:::The file \(self.videoOutputFullFileName!) doesn't exist")
                    }

                    do {
                        try fileManager.copyItem(at: URL(fileURLWithPath: self.videoOutputFullFileName!), to: URL(fileURLWithPath: finalFilename))
                    }
                    catch let error as NSError {
                        NSLog("ERROR:::\(error.debugDescription)")
                    }

                    PHPhotoLibrary.shared().performChanges({
                        PHAssetCollectionChangeRequest.creationRequestForAssetCollection(withTitle: "xxx")
                        PHAssetChangeRequest.creationRequestForAssetFromVideo(atFileURL: URL(fileURLWithPath: finalFilename))
                    }) { completed, error in
                        if completed {
                            NSLog("Video \(self.videoOutputFullFileName ?? "") has been moved to camera roll")
                        }

                        if error != nil {
                            NSLog("ERROR:::Cannot move the video \(self.videoOutputFullFileName ?? "") to camera roll, error: \(error!.localizedDescription)")
                        }

                        finishedWriting = true
                    }

                } else {
                    NSLog("ERROR:::The file: \(self.videoOutputFullFileName ?? "") doesn't exist, so can't move this file camera roll")
                    finishedWriting = true
                }
            } else {
                NSLog("WARN:::The videoWriter status is not completed, status: \(videoWriter.status)")
                finishedWriting = true
            }
        }

        while finishedWriting == false {
    //          NSLog("DEBUG:::Waiting to finish writing...")
        }

我认为你也需要在某个时候调用extensionContext.completeRequest，但是我的代码没有调用也能正常工作耸肩。

- Marty

---

我正在尝试将录制的视频保存到摄像机胶卷中，但似乎不起作用。我已经检查了录制URL中的内容，通过转换为数据来验证，现在我正在调用 **PHPhotoLibrary.shared().performChanges({**，然后是完成处理程序，但无论成功还是失败都没有被触发。你能帮帮我吗？ - McDonal_11

这实际上应该使用DispatchGroup而不是while循环。你是在阻止方法返回直到达到完成处理程序吗？ - Marty

我尝试了你的DispatchGroup方法，它完美地运行了！让我们用DispatchGroup实现写另一个答案。 - Martin

@Marty，你能帮我一下吗？https://dev59.com/8Ljoa4cB1Zd3GeqPAo42 - Govind Rakholiya

Answer 2

应该接受@Marty的答案，因为他指出了问题并且他的DispatchGroup解决方案完美运作。
由于他使用了一个while 循环，并没有描述如何使用 DispatchGroup，这是我实现它的方法。

override func broadcastFinished() {
    let dispatchGroup = DispatchGroup()
    dispatchGroup.enter()
    self.writerInput.markAsFinished()
    self.writer.finishWriting {
        // Do your work to here to make video available
        dispatchGroup.leave()
    }
    dispatchGroup.wait() // <= blocks the thread here
}

- Martin

Answer 3

你可以尝试这样写：

override func broadcastFinished() {
    Log(#function)
    ...
    // Need to give the end CMTime, if not set, the video cannot be used
    videoWriter.endSession(atSourceTime: ...)
    videoWriter.finishWriting {
        // Callback cannot be executed here
    }
    ...
    // The program has been executed.
}

- zuizhe