iOS使用vImage - 加速转换QCAR YUV到RGB

Question

我正在尝试测试将Vuforia生成的YUV图像转换为UIImage，并使用iOS Accelerate Framework的vImage调用来提高其性能。目前代码处于试验阶段。目前转换出的图像呈现为深色条纹状。是否有关于Vuforia在其实现中如何布局YUV格式的已发布详细信息？我的初步假设是它们使用iOS设备使用的双平面420p格式。以下是相关测试代码。

UIImage *imageWithQCARCameraImage(const QCAR::Image *cameraImage)
{
    UIImage *image = nil;

    if (cameraImage) {
        QCAR::PIXEL_FORMAT pixelFormat = cameraImage->getFormat();

        CGColorSpaceRef colorSpace = NULL;
        switch (pixelFormat) {
            case QCAR::YUV:
            case QCAR::RGB888:
                colorSpace = CGColorSpaceCreateDeviceRGB();
                break;
            case QCAR::GRAYSCALE:
                colorSpace = CGColorSpaceCreateDeviceGray();
                break;
            case QCAR::RGB565:
            case QCAR::RGBA8888:
            case QCAR::INDEXED:
                std::cerr << "Image format conversion not implemented." << std::endl;
                break;
            case QCAR::UNKNOWN_FORMAT:
                std::cerr << "Image format unknown." << std::endl;
                break;
        }

        int bitsPerComponent = 8;
        int width = cameraImage->getWidth();
        int height = cameraImage->getHeight();
        const void *baseAddress = cameraImage->getPixels();
        size_t totalBytes = QCAR::getBufferSize(width, height, pixelFormat);

        CGBitmapInfo bitmapInfo = kCGBitmapByteOrderDefault | kCGImageAlphaNone;
        CGColorRenderingIntent renderingIntent = kCGRenderingIntentDefault;
        CGImageRef imageRef = NULL;

        if (pixelFormat == QCAR::YUV) {
            int bytesPerPixel = 4;
            uint8_t *sourceDataAddress = (uint8_t *)baseAddress;

            static vImage_Buffer srcYp = {
                .width = static_cast<vImagePixelCount>(width),
                .height = static_cast<vImagePixelCount>(height),
                .data = const_cast<void *>(baseAddress)
            };

            size_t lumaBytes = width * height;
            size_t chromianceBytes = totalBytes - lumaBytes;
            static vImage_Buffer srcCb = {
                .data = static_cast<void *>(sourceDataAddress + lumaBytes)
            };

            static vImage_Buffer srcCr = {
                .data = static_cast<void *>(sourceDataAddress + lumaBytes + (chromianceBytes / 2))
            };

            static vImage_Buffer dest = {
                .width = static_cast<vImagePixelCount>(width),
                .height = static_cast<vImagePixelCount>(height),
                .data = imageData
            };

            //uint8_t permuteMap[] = { 1, 2, 3, 0 };
            vImage_YpCbCrPixelRange pixelRange = (vImage_YpCbCrPixelRange){ 0, 128, 255, 255, 255, 1, 255, 0 };
            vImage_YpCbCrToARGB info;

            vImage_Error error;

            error = vImageConvert_YpCbCrToARGB_GenerateConversion(kvImage_YpCbCrToARGBMatrix_ITU_R_601_4,
                                                                  &pixelRange,
                                                                  &info,
                                                                  kvImage420Yp8_Cb8_Cr8,
                                                                  kvImageARGB8888,
                                                                  kvImagePrintDiagnosticsToConsole);


            error = vImageConvert_420Yp8_Cb8_Cr8ToARGB8888(&srcYp,
                                                           &srcCb,
                                                           &srcCr,
                                                           &dest,
                                                           &info,
                                                           NULL,
                                                           1,
                                                           kvImageNoFlags);

            vImage_CGImageFormat format =
            {
                .bitsPerComponent = static_cast<uint32_t>(bitsPerComponent),
                .bitsPerPixel = static_cast<uint32_t>(3 * bitsPerComponent),
                .colorSpace = colorSpace,
                .bitmapInfo = bitmapInfo,
                .version = 0,
                .decode = NULL,
                .renderingIntent = renderingIntent
            };

            imageRef = vImageCreateCGImageFromBuffer(&dest,
                                                     &format,
                                                     NULL,
                                                     NULL,
                                                     kvImageNoFlags,
                                                     &error);
            if (error) {
                std::cerr << "Err." << std::endl;
            }
        } else {
            int bitsPerPixel = QCAR::getBitsPerPixel(pixelFormat);
            int bytesPerRow = cameraImage->getStride();
            CGDataProviderRef provider = CGDataProviderCreateWithData(NULL,
                                                                      baseAddress,
                                                                      totalBytes,
                                                                      NULL);

           imageRef = CGImageCreate(width,
                                    height,
                                    bitsPerComponent,
                                    bitsPerPixel,
                                    bytesPerRow,
                                    colorSpace,
                                    bitmapInfo,
                                    provider,
                                    NULL,
                                    false,
                                    renderingIntent);
            CGDataProviderRelease(provider);
        }

        if (imageRef != NULL) {
            image = [UIImage imageWithCGImage:imageRef];
            CGImageRelease(imageRef);
        }

        if (colorSpace != NULL) {
            CGColorSpaceRelease(colorSpace);
        }

    }

    return image;
}

- Cameron Lowell Palmer

---

到目前为止，高通发布的任何细节都是毫无用处的。但在他们的论坛中，版主会“尝试”回答问题。如果这是一个好问题，他们只会有两种回应：“请参阅文档”或者给出实际答案。希望你能得到你想要的答案，听起来很有趣。 - Augmented Jacob

@IanOllmann，这就是为什么我说只是为了测试。但是，如果你要对功能做出这样的声明，提供引用将会很有帮助；如果你谈论性能，那么这就是为什么我说它只是为了测试。按照这个顺序：让它工作，让它正确，让它快速。 - Cameron Lowell Palmer

我是自己的引用。需要时将提供燃烧的灌木丛。 - Ian Ollmann

一份烧烤串，谢谢。 - Cameron Lowell Palmer

然而，这并没有回答基本问题，即... Vuforia YUV帧的格式是什么？ - Cameron Lowell Palmer

Answer 1

void *baseAddress = buffer;
size_t totalBytes = width * height * 3 / 2;

uint8_t *sourceDataAddress = (uint8_t *)baseAddress;

vImage_Buffer srcYp = {
    .width = static_cast<vImagePixelCount>(width),
    .height = static_cast<vImagePixelCount>(height),
    .rowBytes = static_cast<size_t>(width),
    .data = const_cast<void *>(baseAddress),
};

size_t lumaBytes = width * height;
size_t chromianceBytes = totalBytes - lumaBytes;
vImage_Buffer srcCb = {
    .width = static_cast<vImagePixelCount>(width) / 2,
    .height = static_cast<vImagePixelCount>(height) / 2,
    .rowBytes = static_cast<size_t>(width) / 2,
    .data = static_cast<void *>(sourceDataAddress + lumaBytes),
};

vImage_Buffer srcCr = {
    .width = static_cast<vImagePixelCount>(width) / 2,
    .height = static_cast<vImagePixelCount>(height) / 2,
    .rowBytes = static_cast<size_t>(width) / 2,
    .data = static_cast<void *>(sourceDataAddress + lumaBytes + (chromianceBytes / 2)),
};

vImage_Buffer dest;
dest.data = NULL;
vImage_Error error = kvImageNoError;
error = vImageBuffer_Init(&dest, height, width, 32, kvImageNoFlags);
// vImage_YpCbCrPixelRange pixelRange = (vImage_YpCbCrPixelRange){ 0, 128, 255, 255, 255, 1, 255, 0 };
vImage_YpCbCrPixelRange pixelRange = { 16, 128, 235, 240, 255, 0, 255, 0 };
vImage_YpCbCrToARGB info;
error = kvImageNoError;
error = vImageConvert_YpCbCrToARGB_GenerateConversion(kvImage_YpCbCrToARGBMatrix_ITU_R_601_4,
                                                      &pixelRange,
                                                      &info,
                                                      kvImage420Yp8_Cb8_Cr8,
                                                      kvImageARGB8888,
                                                      kvImagePrintDiagnosticsToConsole);
error = kvImageNoError;
uint8_t permuteMap[4] = {3, 2, 1, 0}; // BGRA - iOS only support BGRA
error = vImageConvert_420Yp8_Cb8_Cr8ToARGB8888(&srcYp,
                                               &srcCb,
                                               &srcCr,
                                               &dest,
                                               &info,
                                               permuteMap, // for iOS must be no NULL, mac can be NULL iOS only support BGRA
                                               255,
                                               kvImageNoFlags);

- jeff

---

[编辑]您的答案并添加解释如何/为什么此代码解决了问题（在上面的代码中，而不是在注释中）。 - barbsan

基本上，OP 的错误在于没有在 srcCb 和 srcCr 中将宽度和高度除以 2。"两个色度平面（蓝色和红色投影）在水平和垂直维度上都按 2 的因子进行亚采样"。请参见 https://wiki.videolan.org/YUV。 - Gibezynu Nu