如何将CVPixelBufferRef转换为openCV cv::Mat？

Question

如何将CVPixelBufferRef转换为openCV cv::Mat？

14

我想对CVPixelBufferRef执行几个操作，并得到一个cv::Mat

裁剪到感兴趣的区域
按比例缩放到固定尺寸
均衡化直方图
转换为灰度图像 - 每像素8位（CV_8UC1）

我不确定最有效的操作顺序是什么，但我知道所有这些操作都可以在open：CV矩阵上进行，因此我想知道如何将其转换。

- (void) captureOutput:(AVCaptureOutput *)captureOutput 
         didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer 
         fromConnection:(AVCaptureConnection *)connection
{
     CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);

     cv::Mat frame = f(pixelBuffer); // how do I implement f()?

- Robert

3个回答

5

我正在使用这个。我的cv:Mat被配置为BGR(8UC3)颜色格式。

CVImageBufferRef -> cv::Mat

- (cv::Mat) matFromImageBuffer: (CVImageBufferRef) buffer {

    cv::Mat mat ;

    CVPixelBufferLockBaseAddress(buffer, 0);

    void *address = CVPixelBufferGetBaseAddress(buffer);
    int width = (int) CVPixelBufferGetWidth(buffer);
    int height = (int) CVPixelBufferGetHeight(buffer);

    mat   = cv::Mat(height, width, CV_8UC4, address, 0);
    //cv::cvtColor(mat, _mat, CV_BGRA2BGR);

    CVPixelBufferUnlockBaseAddress(buffer, 0);

    return mat;
}

将cv::Mat转换为CVImageBufferRef（CVPixelBufferRef）

- (CVImageBufferRef) getImageBufferFromMat: (cv::Mat) mat {

    cv::cvtColor(mat, mat, CV_BGR2BGRA);

    int width = mat.cols;
    int height = mat.rows;

    NSDictionary *options = [NSDictionary dictionaryWithObjectsAndKeys:
                             // [NSNumber numberWithBool:YES], kCVPixelBufferCGImageCompatibilityKey,
                             // [NSNumber numberWithBool:YES], kCVPixelBufferCGBitmapContextCompatibilityKey,
                             [NSNumber numberWithInt:width], kCVPixelBufferWidthKey,
                             [NSNumber numberWithInt:height], kCVPixelBufferHeightKey,
                             nil];

    CVPixelBufferRef imageBuffer;
    CVReturn status = CVPixelBufferCreate(kCFAllocatorMalloc, width, height, kCVPixelFormatType_32BGRA, (CFDictionaryRef) CFBridgingRetain(options), &imageBuffer) ;


    NSParameterAssert(status == kCVReturnSuccess && imageBuffer != NULL);

    CVPixelBufferLockBaseAddress(imageBuffer, 0);
    void *base = CVPixelBufferGetBaseAddress(imageBuffer) ;
    memcpy(base, mat.data, _mat.total()*4);
    CVPixelBufferUnlockBaseAddress(imageBuffer, 0);

    return imageBuffer;
}

- newinH

self.rows是什么？为什么不是mat.rows？另外，如何从这个CVImageBufferRef创建CMSampleBufferRef？ - user924

哦，它有一些错误。 - newinH

函数开头的"-"代表什么意思？ - Amacelia

2

@Amacelia 它表示这个函数是Objective C中的实例方法，而不是类方法。 - newinH

此答案假定相机 videoSettings 正在使用 [ String(kCVPixelBufferPixelFormatTypeKey) : kCMPixelFormat_32BGRA]。此外，在苹果手机的后续版本中，相机返回的 CVPixelBuffer 在每行末尾会有额外的填充。因此，为了向前兼容，cv::Mat 定义应改为 mat = cv::Mat(height, width, CV_8UC4, address, baseaddress, CVPixelBufferGetBytesPerRow(buffer));。 - Drew H

0

对于正在寻找新的OpenCV 4.4 Swift Wrapper的Swift代码的人们：

// Converts CVPixelBuffer to Mat object
/// - Parameter pixelBuffer: BGRA pixel data
/// - Returns: BGRA img data as a Mat object
func cvPixelBufferToMat(pixelBuffer: CVPixelBuffer)-> Mat? {
    
    CVPixelBufferLockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
    
    let base = CVPixelBufferGetBaseAddress(pixelBuffer)
    let width = Int32(CVPixelBufferGetWidth(pixelBuffer))
    let height = Int32(CVPixelBufferGetHeight(pixelBuffer))
    
    let matrix = Mat(rows: width, cols: height, type: CvType.CV_8UC4)
    memcpy(matrix.dataPointer(), base, CVPixelBufferGetDataSize(pixelBuffer))
    
    CVPixelBufferUnlockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
    
    return matrix
}



// Converts Mat object to CVPixelBuffer
/// - Parameter mat: BGRA img data
/// - Returns: CVPixelBuffer(BGRA)
func matToCVPixelBuffer(mat: Mat)-> CVPixelBuffer? {
    let matrix = mat
    
    let attributes = [
        kCVPixelBufferMetalCompatibilityKey: kCFBooleanTrue!,
        kCVPixelBufferCGImageCompatibilityKey: kCFBooleanTrue!,
        kCVPixelBufferCGBitmapContextCompatibilityKey: kCFBooleanTrue!,
        kCVPixelBufferWidthKey: matrix.cols(),
        kCVPixelBufferHeightKey: matrix.rows(),
        kCVPixelBufferBytesPerRowAlignmentKey: matrix.step1(0)
    ] as CFDictionary
    
    var pixelBuffer: CVPixelBuffer?
    
    let status = CVPixelBufferCreate(
        kCFAllocatorDefault, Int(matrix.cols()),
        Int(matrix.rows()),
        kCVPixelFormatType_32BGRA,
        attributes,
        &pixelBuffer)
    
    guard let pixelBuffer = pixelBuffer, (status == kCVReturnSuccess) else {
        return nil
    }
    
    CVPixelBufferLockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
    
    let base = CVPixelBufferGetBaseAddress(pixelBuffer)
    memcpy(base, matrix.dataPointer(), matrix.total()*matrix.elemSize())
    
    CVPixelBufferUnlockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
    
    return pixelBuffer
}

- Mohamed Salah

网页内容由stack overflow 提供, 点击上面的

可以查看英文原文，
原文链接

- Robert · Accepted Answer

我在一些优秀的GitHub源代码中找到了答案。我为了简单起见对其进行了修改。它还为我执行了灰度转换。

CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
OSType format = CVPixelBufferGetPixelFormatType(pixelBuffer);

// Set the following dict on AVCaptureVideoDataOutput's videoSettings to get YUV output
// @{ kCVPixelBufferPixelFormatTypeKey : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange }

NSAssert(format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, @"Only YUV is supported");

// The first plane / channel (at index 0) is the grayscale plane
// See more infomation about the YUV format
// http://en.wikipedia.org/wiki/YUV
CVPixelBufferLockBaseAddress(pixelBuffer, 0);
void *baseaddress = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);

CGFloat width = CVPixelBufferGetWidth(pixelBuffer);
CGFloat height = CVPixelBufferGetHeight(pixelBuffer);

cv::Mat mat(height, width, CV_8UC1, baseaddress, 0);

// Use the mat here

CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);

我认为最好的顺序是：

转换为灰度（因为这几乎是自动完成的）
裁剪（这应该是一个快速操作，并且可以减少要处理的像素数）
缩小比例
均衡直方图