我正在使用Vision框架和CoreML在增强现实应用中，通过ARSession代理对每个帧的图像进行分类，在ARKit和RealityKit中。在处理frame.capturedImage时，为了提高性能，我不会请求另一个frame.capturedImage。
相机没有提供流畅的体验，它时不时地卡顿。似乎是丢帧了。
并且我收到了这个警告：
[Session] ARSession <0x122cc3710>: ARSessionDelegate is retaining 14 ARFrames. This can lead to future camera frames being dropped.
我的代码：
import Foundation
import SwiftUI
import RealityKit
import ARKit
import CoreML
struct ARViewContainer: UIViewRepresentable {
var errorFunc: ()->Void
var frameUpdateFunc: ()->Void
@Binding var finalLabel:String
func makeUIView(context: Context) -> ARView {
let arView = ARView(frame: .zero)
let config = ARWorldTrackingConfiguration()
config.planeDetection = [.horizontal,.vertical]
config.environmentTexturing = .automatic
if ARWorldTrackingConfiguration.supportsSceneReconstruction(.mesh){
config.sceneReconstruction = .mesh
}
arView.session.delegate = context.coordinator
arView.session.run(config)
context.coordinator.myView = arView
return arView
}
func updateUIView(_ uiView: ARView, context: Context) {
}
func makeCoordinator() -> Coordinator {
Coordinator(finalLabel: $finalLabel, self, funct: self.errorFunc, frameUpdateFunc: self.frameUpdateFunc)
}
class Coordinator: NSObject, ARSessionDelegate {
var objectDetectionService = ObjectDetectionService()
var myView:ARView?
@Binding var finalLabel:String
var parent: ARViewContainer
var efunc:()->Void
var frameUpdateFunc:()->Void
var isLoopShouldContinue = true
var lastLocation: SCNVector3?
//let model = try? MobileNetV2(configuration: .init())
private let classifier = VisionClasifier(mlModel: try? MobileNetV2(configuration: .init()).model)
private var currentBuffer: CVPixelBuffer? = nil
init(finalLabel:Binding<String>,_ arView: ARViewContainer,funct: @escaping ()->Void, frameUpdateFunc:@escaping ()->Void) {
parent = arView
self.efunc = funct
self.frameUpdateFunc = frameUpdateFunc
_finalLabel = finalLabel
}
func session(_ session: ARSession, didFailWithError error: Error) {
//print("Error Tanvir: ",error)
self.efunc()
}
func session(_ session: ARSession, didUpdate frame: ARFrame) {
if isLoopShouldContinue{
self.classifyFrame(currentFrame: frame)
}
let transform = SCNMatrix4(frame.camera.transform)
let orientation = SCNVector3(-transform.m31, -transform.m32, transform.m33)
let location = SCNVector3(transform.m41, transform.m42, transform.m43)
let currentPositionOfCamera = orientation + location
if let lastLocation = lastLocation {
let speed = (lastLocation - currentPositionOfCamera).length()
isLoopShouldContinue = speed < 0.0025
}
lastLocation = currentPositionOfCamera
}
// When ARKit detects a new anchor, it will add it to the ARSession
// Whenever there is a newly added ARAnchor, you will get that anchor here.
// In this short tutorial, we will target the ARPlaneAnchor, and use the information stored
// in that anchor for visualization.
func session(_ session: ARSession, didAdd anchors: [ARAnchor]) {
guard let myView = myView else {
return
}
for anchor in anchors {
if anchor is ARPlaneAnchor {
let planeAnchor = anchor as! ARPlaneAnchor
//addPlaneEntity(with: planeAnchor, to: myView)
}
}
}
// ARKit will automatically track and update the ARPlaneAnchor.
// We use that anchor to update the `skin` of the plane.
func session(_ session: ARSession, didUpdate anchors: [ARAnchor]) {
guard let myView = myView else {
return
}
for anchor in anchors {
if anchor is ARPlaneAnchor {
let planeAnchor = anchor as! ARPlaneAnchor
//updatePlaneEntity(with: planeAnchor, in: myView)
}
}
}
// When ARKit remove an anchor from the ARSession, you will get the removed
// anchor here.
func session(_ session: ARSession, didRemove anchors: [ARAnchor]) {
guard let myView = myView else {
return
}
for anchor in anchors {
if anchor is ARPlaneAnchor {
let planeAnchor = anchor as! ARPlaneAnchor
//removePlaneEntity(with: planeAnchor, from: myView)
}
}
}
func addAnnotation(rectOfInterest rect: CGRect, text: String,width:Float,height:Float) {
let point = CGPoint(x: rect.midX, y: rect.midY)
print("point:", point)
//let scnHitTestResults = myView.hitTest(point,
// options: [SCNHitTestOption.searchMode: SCNHitTestSearchMode.all.rawValue])
//guard !scnHitTestResults.contains(where: { $0.node.name == BubbleNode.name }) else { return }
let raycastResult = myView!.raycast(from: point, allowing: .estimatedPlane, alignment: .any)
// guard let raycastQuery = myView!.raycastQuery(from: point,
// allowing: .existingPlaneInfinite,
// alignment: .horizontal),
// let raycastResult = myView.session.raycast(raycastQuery).first else { return }
guard let raycastResult = raycastResult.first else{
print("raycast result failed")
return
}
let anchorExists = myView!.scene.anchors.contains(where: {$0.name == text})
guard anchorExists == false else{
print("anchor Already exists")
return
}
let position = raycastResult.worldTransform.columns.3
let myEntity = create2dEntity(with: position, boundingBox: rect, raycastResult: raycastResult,width:width ,height:height)
let planeAnchorEntity = AnchorEntity()
planeAnchorEntity.name = text
planeAnchorEntity.position = simd_make_float3(position)
planeAnchorEntity.addChild(myEntity)
// Finally, add the entity to scene.
myView!.scene.addAnchor(planeAnchorEntity)
print("anchor added: ", planeAnchorEntity.name)
}
func classifyFrame(currentFrame:ARFrame){
//let currentImageName = photos[currentIndex]
// 2
// 3
print("inside Classify")
//print("CurrentBuffer", currentBuffer)
guard self.currentBuffer == nil else {
//print("CurrentBuffer: ",currentBuffer)
//self.finalLabel = "current buffer problem"
return
}
self.currentBuffer = currentFrame.capturedImage
// guard let model = self.model else {
// return "Model not Found."
// }
let img = CIImage(cvImageBuffer: currentFrame.capturedImage)
let cgImage = convertCIImageToCGImage(inputImage: img)
guard let cgImage = cgImage else{
print("can not convert CGImage")
self.finalLabel = "can not convert CGImage"
return
}
objectDetectionService.detect(on: .init(pixelBuffer: currentFrame.capturedImage)) { [weak self] result in
guard let self = self else { return }
switch result {
case .success(let response):
self.finalLabel = response.classification.description
print("Real Width: ",response.boundingBox.width)
let rectOfInterest = VNImageRectForNormalizedRect(
response.boundingBox,
Int(self.myView!.bounds.width),
Int(self.myView!.bounds.height))
self.addAnnotation(rectOfInterest: rectOfInterest, text: response.classification.description,width: Float(response.boundingBox.width),height: Float(response.boundingBox.height))
print("Success:",response.classification.description)
self.currentBuffer = nil
case .failure(let error):
self.finalLabel = "Detection Failed"
print("Detection failure: ",error.localizedDescription)
self.currentBuffer = nil
break
}
}
}
}
}
func convertCIImageToCGImage(inputImage: CIImage) -> CGImage? {
let context = CIContext(options: nil)
if let cgImage = context.createCGImage(inputImage, from: inputImage.extent) {
return cgImage
}
return nil
}
// The ARPlaneAnchor contains the information we need to create the `skin` of the plane.
func addPlaneEntity(with anchor: ARPlaneAnchor, to view: ARView) {
let planeAnchorEntity = AnchorEntity(.plane([.any],
classification: [.any],
minimumBounds: [0.01, 0.01]))
let planeModelEntity = createPlaneModelEntity(with: anchor)
// Give Entity a name for tracking.
planeAnchorEntity.name = anchor.identifier.uuidString + "_anchor"
planeModelEntity.name = anchor.identifier.uuidString + "_model"
// Add ModelEntity as a child of AnchorEntity.
// AnchorEntity handles `position` of the plane.
// ModelEntity handles the `skin` of the plane.
planeAnchorEntity.addChild(planeModelEntity)
// Finally, add the entity to scene.
view.scene.addAnchor(planeAnchorEntity)
}
func create2dEntity(with position: simd_float4, boundingBox: CGRect, raycastResult:ARRaycastResult, width:Float,height:Float ) -> ModelEntity{
var planeMesh: MeshResource
var color: UIColor
print("horizotal plane")
color = UIColor.red.withAlphaComponent(0.5)
print("Constant width: 0.1 but BoundingBox Width: ",boundingBox.width)
planeMesh = .generatePlane(width: 0.1, height: 0.1)
return ModelEntity(mesh: planeMesh, materials: [SimpleMaterial(color: color, roughness: 0.25, isMetallic: false)])
}
func createPlaneModelEntity(with anchor: ARPlaneAnchor) -> ModelEntity {
var planeMesh: MeshResource
var color: UIColor
if anchor.alignment == .horizontal {
print("horizotal plane")
color = UIColor.blue.withAlphaComponent(0.5)
planeMesh = .generatePlane(width: anchor.extent.x, depth: anchor.extent.z)
} else if anchor.alignment == .vertical {
print("vertical plane")
color = UIColor.yellow.withAlphaComponent(0.5)
planeMesh = .generatePlane(width: anchor.extent.x, height: anchor.extent.z)
} else {
fatalError("Anchor is not ARPlaneAnchor")
}
return ModelEntity(mesh: planeMesh, materials: [SimpleMaterial(color: color, roughness: 0.25, isMetallic: false)])
}
func removePlaneEntity(with anchor: ARPlaneAnchor, from arView: ARView) {
guard let planeAnchorEntity = arView.scene.findEntity(named: anchor.identifier.uuidString+"_anchor") else { return }
arView.scene.removeAnchor(planeAnchorEntity as! AnchorEntity)
}
func updatePlaneEntity(with anchor: ARPlaneAnchor, in view: ARView) {
var planeMesh: MeshResource
guard let entity = view.scene.findEntity(named: anchor.identifier.uuidString+"_model") else { return }
let modelEntity = entity as! ModelEntity
if anchor.alignment == .horizontal {
planeMesh = .generatePlane(width: anchor.extent.x, depth: anchor.extent.z)
} else if anchor.alignment == .vertical {
planeMesh = .generatePlane(width: anchor.extent.x, height: anchor.extent.z)
} else {
fatalError("Anchor is not ARPlaneAnchor")
}
modelEntity.model!.mesh = planeMesh
}
import SceneKit
extension SCNVector3 {
func length() -> Float {
return sqrtf(x * x + y * y + z * z)
}
}
func -(l: SCNVector3, r: SCNVector3) -> SCNVector3 {
return SCNVector3Make(l.x - r.x, l.y - r.y, l.z - r.z)
}
func +(l: SCNVector3, r: SCNVector3) -> SCNVector3 {
return SCNVector3(l.x + r.x, l.y + r.y, l.z + r.z)
}
func /(l: SCNVector3, r: Float) -> SCNVector3 {
return SCNVector3(l.x / r, l.y / r, l.z / r)
}
检测：（我猜在detect方法中存在问题）
import Foundation
import UIKit
import CoreML
import Vision
import SceneKit
class ObjectDetectionService {
var mlModel = try! VNCoreMLModel(for: YOLOv3Int8LUT().model)
//let model = try? YOLOv3Int8LUT(configuration: .init())
lazy var coreMLRequest: VNCoreMLRequest = {
return VNCoreMLRequest(model: mlModel,
completionHandler: self.coreMlRequestHandler)
}()
private var completion: ((Result<Response, Error>) -> Void)?
func detect(on request: Request, completion: @escaping (Result<Response, Error>) -> Void) {
self.completion = completion
//let orientation = .up
let imageRequestHandler = VNImageRequestHandler(cvPixelBuffer: request.pixelBuffer)
do {
try imageRequestHandler.perform([coreMLRequest])
} catch {
self.complete(.failure(error))
return
}
}
}
private extension ObjectDetectionService {
func coreMlRequestHandler(_ request: VNRequest?, error: Error?) {
if let error = error {
complete(.failure(error))
return
}
guard let request = request, let results = request.results as? [VNRecognizedObjectObservation] else {
complete(.failure(RecognitionError.resultIsEmpty))
return
}
guard let result = results.first(where: { $0.confidence > 0.8 }),
let classification = result.labels.first else {
complete(.failure(RecognitionError.lowConfidence))
return
}
let response = Response(boundingBox: result.boundingBox,
classification: classification.identifier)
complete(.success(response))
}
func complete(_ result: Result<Response, Error>) {
DispatchQueue.main.async {
self.completion?(result)
self.completion = nil
}
}
}
enum RecognitionError: Error {
case unableToInitializeCoreMLModel
case resultIsEmpty
case lowConfidence
}
extension ObjectDetectionService {
struct Request {
let pixelBuffer: CVPixelBuffer
}
struct Response {
let boundingBox: CGRect
let classification: String
}
}
为什么会出现这个警告，如何获得更流畅的相机体验？