kCVPixelFormatType_420YpCbCr8BiPlanarFullRange frame to UIImage conversion

Question

I have an app that capture live video in kCVPixelFormatType_420YpCbCr8BiPlanarFullRange format to process Y channel. According to Apple\'s documentation:

Answer 1

These other answers with the bit shifting and the magic variables are wild. Here's an alternative approach using the Accelerate framework in Swift 5. It takes a frame from a buffer with pixel format kCVPixelFormatType_420YpCbCr8BiPlanarFullRange (Bi-Planar Component Y'CbCr 8-bit 4:2:0) and makes a UIImage from it after converting it to ARGB8888. But you could probably modify it to handle any input/output formats:

import Accelerate
import CoreGraphics
import CoreMedia
import Foundation
import QuartzCore
import UIKit

func createImage(from sampleBuffer: CMSampleBuffer) -> UIImage? {
    guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
      return nil
    }

    // pixel format is Bi-Planar Component Y'CbCr 8-bit 4:2:0, full-range (luma=[0,255] chroma=[1,255]).
    // baseAddr points to a big-endian CVPlanarPixelBufferInfo_YCbCrBiPlanar struct.
    //
    guard CVPixelBufferGetPixelFormatType(imageBuffer) == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange else {
        return nil
    }


    guard CVPixelBufferLockBaseAddress(imageBuffer, .readOnly) == kCVReturnSuccess else {
        return nil
    }

    defer {
        // be sure to unlock the base address before returning
        CVPixelBufferUnlockBaseAddress(imageBuffer, .readOnly)
    }

    // 1st plane is luminance, 2nd plane is chrominance
    guard CVPixelBufferGetPlaneCount(imageBuffer) == 2 else {
        return nil
    }

    // 1st plane
    guard let lumaBaseAddress = CVPixelBufferGetBaseAddressOfPlane(imageBuffer, 0) else {
        return nil
    }

    let lumaWidth = CVPixelBufferGetWidthOfPlane(imageBuffer, 0)
    let lumaHeight = CVPixelBufferGetHeightOfPlane(imageBuffer, 0)
    let lumaBytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(imageBuffer, 0)
    var lumaBuffer = vImage_Buffer(
        data: lumaBaseAddress,
        height: vImagePixelCount(lumaHeight),
        width: vImagePixelCount(lumaWidth),
        rowBytes: lumaBytesPerRow
    )

    // 2nd plane
    guard let chromaBaseAddress = CVPixelBufferGetBaseAddressOfPlane(imageBuffer, 1) else {
        return nil
    }

    let chromaWidth = CVPixelBufferGetWidthOfPlane(imageBuffer, 1)
    let chromaHeight = CVPixelBufferGetHeightOfPlane(imageBuffer, 1)
    let chromaBytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(imageBuffer, 1)
    var chromaBuffer = vImage_Buffer(
        data: chromaBaseAddress,
        height: vImagePixelCount(chromaHeight),
        width: vImagePixelCount(chromaWidth),
        rowBytes: chromaBytesPerRow
    )

    var argbBuffer = vImage_Buffer()

    defer {
        // we are responsible for freeing the buffer data
        free(argbBuffer.data)
    }

    // initialize the empty buffer
    guard vImageBuffer_Init(
        &argbBuffer,
        lumaBuffer.height,
        lumaBuffer.width,
        32,
        vImage_Flags(kvImageNoFlags)
        ) == kvImageNoError else {
            return nil
    }

    // full range 8-bit, clamped to full range, is necessary for correct color reproduction
    var pixelRange = vImage_YpCbCrPixelRange(
        Yp_bias: 0,
        CbCr_bias: 128,
        YpRangeMax: 255,
        CbCrRangeMax: 255,
        YpMax: 255,
        YpMin: 1,
        CbCrMax: 255,
        CbCrMin: 0
    )

    var conversionInfo = vImage_YpCbCrToARGB()

    // initialize the conversion info
    guard vImageConvert_YpCbCrToARGB_GenerateConversion(
        kvImage_YpCbCrToARGBMatrix_ITU_R_601_4, // Y'CbCr-to-RGB conversion matrix for ITU Recommendation BT.601-4.
        &pixelRange,
        &conversionInfo,
        kvImage420Yp8_CbCr8, // converting from
        kvImageARGB8888, // converting to
        vImage_Flags(kvImageNoFlags)
        ) == kvImageNoError else {
            return nil
    }

    // do the conversion
    guard vImageConvert_420Yp8_CbCr8ToARGB8888(
        &lumaBuffer, // in
        &chromaBuffer, // in
        &argbBuffer, // out
        &conversionInfo,
        nil,
        255,
        vImage_Flags(kvImageNoFlags)
        ) == kvImageNoError else {
            return nil
    }

    // core foundation objects are automatically memory mananged. no need to call CGContextRelease() or CGColorSpaceRelease()
    guard let context = CGContext(
        data: argbBuffer.data,
        width: Int(argbBuffer.width),
        height: Int(argbBuffer.height),
        bitsPerComponent: 8,
        bytesPerRow: argbBuffer.rowBytes,
        space: CGColorSpaceCreateDeviceRGB(),
        bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue
        ) else {
            return nil
    }

    guard let cgImage = context.makeImage() else {
        return nil
    }

    return UIImage(cgImage: cgImage) 
}

Answer 2

I'm not aware of any accessible built-in way to convert a biplanar Y / CbCr image to RGB in iOS. However you should be able to perform the conversion yourself in software, e.g.

uint8_t clamp(int16_t input)
{
    // clamp negative numbers to 0; assumes signed shifts
    // (a valid assumption on iOS)
    input &= ~(num >> 16);

    // clamp numbers greater than 255 to 255; the accumulation
    // of the mask looks odd but is an attempt to avoid
    // pipeline stalls
    uint8_t saturationMask = num >> 8;
    saturationMask |= saturationMask << 4;
    saturationMask |= saturationMask << 2;
    saturationMask |= saturationMask << 1;
    num |= saturationMask;

    return num&0xff;
}

...

CVPixelBufferLockBaseAddress(imageBuffer, 0);

size_t width = CVPixelBufferGetWidth(imageBuffer);
size_t height = CVPixelBufferGetHeight(imageBuffer);

uint8_t *baseAddress = CVPixelBufferGetBaseAddress(imageBuffer);
CVPlanarPixelBufferInfo_YCbCrBiPlanar *bufferInfo = (CVPlanarPixelBufferInfo_YCbCrBiPlanar *)baseAddress;

NSUInteger yOffset = EndianU32_BtoN(bufferInfo->componentInfoY.offset);
NSUInteger yPitch = EndianU32_BtoN(bufferInfo->componentInfoY.rowBytes);

NSUInteger cbCrOffset = EndianU32_BtoN(bufferInfo->componentInfoCbCr.offset);
NSUInteger cbCrPitch = EndianU32_BtoN(bufferInfo->componentInfoCbCr.rowBytes);

uint8_t *rgbBuffer = malloc(width * height * 3);
uint8_t *yBuffer = baseAddress + yOffset;
uint8_t *cbCrBuffer = baseAddress + cbCrOffset;

for(int y = 0; y < height; y++)
{
    uint8_t *rgbBufferLine = &rgbBuffer[y * width * 3];
    uint8_t *yBufferLine = &yBuffer[y * yPitch];
    uint8_t *cbCrBufferLine = &cbCrBuffer[(y >> 1) * cbCrPitch];

    for(int x = 0; x < width; x++)
    {
        // from ITU-R BT.601, rounded to integers
        uint8_t y = yBufferLine[x] - 16;
        uint8_t cb = cbCrBufferLine[x & ~1] - 128;
        uint8_t cr = cbCrBufferLine[x | 1] - 128;

        uint8_t *rgbOutput = &rgbBufferLine[x*3];

        rgbOutput[0] = clamp(((298 * y + 409 * cr - 223) >> 8) - 223);
        rgbOutput[1] = clamp(((298 * y - 100 * cb - 208 * cr + 136) >> 8) + 136);
        rgbOutput[2] = clamp(((298 * y + 516 * cb - 277) >> 8) - 277);
    }

}

Just written directly into this box and untested, I think I've got the cb/cr extraction correct. You'd then use CGBitmapContextCreate with rgbBuffer to create a CGImage and hence a UIImage.

Answer 3

Most implementations I found (including the previous answer here) won't work if you change videoOrientation in the AVCaptureConnection (for some reason I don't fully understand, the CVPlanarPixelBufferInfo_YCbCrBiPlanar struct will be empty in that case), so I wrote one that does (most of the code was based on this answer). My implementation also adds an empty alpha channel to the RGB buffer and creates the CGBitmapContext using the kCGImageAlphaNoneSkipLast flag (there's no alpha data, but iOS seems to require 4 bytes per pixel). Here it is:

#define clamp(a) (a>255?255:(a<0?0:a))

- (UIImage *)imageFromSampleBuffer:(CMSampleBufferRef)sampleBuffer {
    CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
    CVPixelBufferLockBaseAddress(imageBuffer,0);

    size_t width = CVPixelBufferGetWidth(imageBuffer);
    size_t height = CVPixelBufferGetHeight(imageBuffer);
    uint8_t *yBuffer = CVPixelBufferGetBaseAddressOfPlane(imageBuffer, 0);
    size_t yPitch = CVPixelBufferGetBytesPerRowOfPlane(imageBuffer, 0);
    uint8_t *cbCrBuffer = CVPixelBufferGetBaseAddressOfPlane(imageBuffer, 1);
    size_t cbCrPitch = CVPixelBufferGetBytesPerRowOfPlane(imageBuffer, 1);

    int bytesPerPixel = 4;
    uint8_t *rgbBuffer = malloc(width * height * bytesPerPixel);

    for(int y = 0; y < height; y++) {
        uint8_t *rgbBufferLine = &rgbBuffer[y * width * bytesPerPixel];
        uint8_t *yBufferLine = &yBuffer[y * yPitch];
        uint8_t *cbCrBufferLine = &cbCrBuffer[(y >> 1) * cbCrPitch];

        for(int x = 0; x < width; x++) {
            int16_t y = yBufferLine[x];
            int16_t cb = cbCrBufferLine[x & ~1] - 128;
            int16_t cr = cbCrBufferLine[x | 1] - 128;

            uint8_t *rgbOutput = &rgbBufferLine[x*bytesPerPixel];

            int16_t r = (int16_t)roundf( y + cr *  1.4 );
            int16_t g = (int16_t)roundf( y + cb * -0.343 + cr * -0.711 );
            int16_t b = (int16_t)roundf( y + cb *  1.765);

            rgbOutput[0] = 0xff;
            rgbOutput[1] = clamp(b);
            rgbOutput[2] = clamp(g);
            rgbOutput[3] = clamp(r);
        }
    }

    CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
    CGContextRef context = CGBitmapContextCreate(rgbBuffer, width, height, 8, width * bytesPerPixel, colorSpace, kCGBitmapByteOrder32Little | kCGImageAlphaNoneSkipLast);
    CGImageRef quartzImage = CGBitmapContextCreateImage(context);
    UIImage *image = [UIImage imageWithCGImage:quartzImage];

    CGContextRelease(context);
    CGColorSpaceRelease(colorSpace);
    CGImageRelease(quartzImage);
    free(rgbBuffer);

    CVPixelBufferUnlockBaseAddress(imageBuffer, 0);

    return image;
}