在iOS 12上过滤深度数据似乎被旋转了。

Question

我遇到了一个问题，当 isFilteringEnabled = true 时，.builtInDualCamera 的深度数据似乎旋转了90度。

以下是我的代码：

fileprivate let session = AVCaptureSession()

fileprivate let meta = AVCaptureMetadataOutput()
fileprivate let video = AVCaptureVideoDataOutput()
fileprivate let depth = AVCaptureDepthDataOutput()

fileprivate let camera: AVCaptureDevice
fileprivate let input: AVCaptureDeviceInput

fileprivate let synchronizer: AVCaptureDataOutputSynchronizer

init(delegate: CaptureSessionDelegate?) throws {
    self.delegate = delegate
    session.sessionPreset = .vga640x480

    // Setup Camera Input
    let discovery = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInDualCamera], mediaType: .video, position: .unspecified)
    if let device = discovery.devices.first {
        camera = device
    } else {
        throw SessionError.CameraNotAvailable("Unable to load camera")
    }

    input = try AVCaptureDeviceInput(device: camera)
    session.addInput(input)

    // Setup Metadata Output (Face)
    session.addOutput(meta)
    if meta.availableMetadataObjectTypes.contains(AVMetadataObject.ObjectType.face) {
        meta.metadataObjectTypes = [ AVMetadataObject.ObjectType.face ]
    } else {
        print("Can't Setup Metadata: \(meta.availableMetadataObjectTypes)")
    }

    // Setup Video Output
    video.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]
    session.addOutput(video)
    video.connection(with: .video)?.videoOrientation = .portrait

    // ****** THE ISSUE IS WITH THIS BLOCK HERE ******
    // Setup Depth Output
    depth.isFilteringEnabled = true
    session.addOutput(depth)
    depth.connection(with: .depthData)?.videoOrientation = .portrait

    // Setup Synchronizer
    synchronizer = AVCaptureDataOutputSynchronizer(dataOutputs: [depth, video, meta])


    let outputRect = CGRect(x: 0, y: 0, width: 1, height: 1)
    let videoRect = video.outputRectConverted(fromMetadataOutputRect: outputRect)
    let depthRect = depth.outputRectConverted(fromMetadataOutputRect: outputRect)

    // Ratio of the Depth to Video
    scale = max(videoRect.width, videoRect.height) / max(depthRect.width, depthRect.height)

    // Set Camera to the framerate of the Depth Data Collection
    try camera.lockForConfiguration()
    if let fps = camera.activeDepthDataFormat?.videoSupportedFrameRateRanges.first?.minFrameDuration {
        camera.activeVideoMinFrameDuration = fps
    }
    camera.unlockForConfiguration()

    super.init()
    synchronizer.setDelegate(self, queue: syncQueue)
}

func dataOutputSynchronizer(_ synchronizer: AVCaptureDataOutputSynchronizer, didOutput data: AVCaptureSynchronizedDataCollection) {
    guard let delegate = self.delegate else {
        return
    }

    // Check to see if all the data is actually here
    guard
        let videoSync = data.synchronizedData(for: video) as? AVCaptureSynchronizedSampleBufferData,
        !videoSync.sampleBufferWasDropped,
        let depthSync = data.synchronizedData(for: depth) as? AVCaptureSynchronizedDepthData,
        !depthSync.depthDataWasDropped
    else {
        return
    }

    // It's OK if the face isn't found.
    let face: AVMetadataFaceObject?
    if let metaSync = data.synchronizedData(for: meta) as? AVCaptureSynchronizedMetadataObjectData {
            face = (metaSync.metadataObjects.first { $0 is AVMetadataFaceObject }) as? AVMetadataFaceObject
    } else {
            face = nil
    }

    // Convert Buffers to CIImage
    let videoImage = convertVideoImage(fromBuffer: videoSync.sampleBuffer)
    let depthImage = convertDepthImage(fromData: depthSync.depthData, andFace: face)

    // Call Delegate
    delegate.captureImages(video: videoImage, depth: depthImage, face: face)
}

fileprivate func convertVideoImage(fromBuffer sampleBuffer: CMSampleBuffer) -> CIImage {
    // Convert from "CoreMovie?" to CIImage - fairly straight-forward
    let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)
    let image = CIImage(cvPixelBuffer: pixelBuffer!)
    return image
}

fileprivate func convertDepthImage(fromData depthData: AVDepthData, andFace face: AVMetadataFaceObject?) -> CIImage {

    var convertedDepth: AVDepthData

    // Convert 16-bif floats up to 32
    if depthData.depthDataType != kCVPixelFormatType_DisparityFloat32 {
        convertedDepth = depthData.converting(toDepthDataType: kCVPixelFormatType_DisparityFloat32)
    } else {
        convertedDepth = depthData
    }

    // Pixel buffer comes straight from depthData
    let pixelBuffer = convertedDepth.depthDataMap

    let image = CIImage(cvPixelBuffer: pixelBuffer)
    return image
}

原始视频长这样：（供参考）

当值为：

// Setup Depth Output
depth.isFilteringEnabled = false
depth.connection(with: .depthData)?.videoOrientation = .portrait

这张图片看起来是这样的：（你可以看到更近的夹克是白色的，更远的夹克是灰色的，距离是深灰色的 - 正如预期的那样）

当值为：

// Setup Depth Output
depth.isFilteringEnabled = true
depth.connection(with: .depthData)?.videoOrientation = .portrait

这张图片看起来像这样：（您可以看到颜色值似乎在正确的位置，但平滑滤镜中的形状似乎旋转了）

当值为：

// Setup Depth Output
depth.isFilteringEnabled = true
depth.connection(with: .depthData)?.videoOrientation = .landscapeRight

这张图片看起来像这样：（颜色和形状都呈水平状）

我做错了什么导致这些不正确的值？

我已经尝试重新排列代码。

// Setup Depth Output
depth.connection(with: .depthData)?.videoOrientation = .portrait
depth.isFilteringEnabled = true

但是那样并没有任何作用。

我认为这是与iOS 12相关的问题，因为我记得在iOS 11下它完全正常工作（尽管我没有保存任何图片来证明）

非常感谢您的帮助！

- Stephen Furlani

---

也许你应该先了解iPhone如何保存它们的图像轴。 - antonio yaphiar

过去我有一个使用前置摄像头的ARKit项目，当我尝试录制视频时，它会旋转我的录制方向。我解决这个问题的方法是对我正在处理的缓冲区应用变换。因此，在您的情况下，也许您可以有另一个标志isFiltering，当您想要查看经过滤波的图像时设置该标志。您可以在ConvertVideoImage或ConvertDepthImage中使用该标志来旋转CIImage以正确显示它在您的应用程序中。 - NFarrell

Answer 1

与其他关于创建后旋转图像的建议不同，我发现并不可行。在 AVDepthData 文档中有一种方法可用于为您完成方向校正。

该方法称为：depthDataByApplyingExifOrientation:，它返回应用了方向的 AVDepthData 实例，即您可以通过传递所需参数来创建正确方向的图像。

这是我的辅助方法，返回已经修复了方向的 UIImage。

- (UIImage *)createDepthMapImageFromCapturePhoto:(AVCapturePhoto *)photo {
    // AVCapturePhoto which has depthData - in swift you should confirm this exists
    AVDepthData *frontDepthData = [photo depthData];
    // Overwrite the instance with the correct orientation applied.
    frontDepthData = [frontDepthData depthDataByApplyingExifOrientation:kCGImagePropertyOrientationRight];
    // Create the CIImage from the depth data using the available method.
    CIImage *ciDepthImage = [CIImage imageWithDepthData:frontDepthData];
    // Create CIContext which enables converting CIImage to CGImage
    CIContext *context = [[CIContext alloc] init];
    // Create the CGImage
    CGImageRef img = [context createCGImage:ciDepthImage fromRect:[ciDepthImage extent]];
    // Create the final image.
    UIImage *depthImage = [UIImage imageWithCGImage:img];
    // Return the depth image.
    return depthImage;
}

- App Dev Guy

---

我正在为一个特定的项目编写Objective-C代码。将其转换成Swift应该不难。 - App Dev Guy

谢谢，不幸的是我已经不在这个项目上工作了，所以无法测试并将您的答案标记为已接受，但我很感激您的工作。 - Stephen Furlani

我可以确认这个有效。 - CrossProduct