Convert Android camera2 api YUV_420_888 to RGB

Question

I am writing an app that takes the camera feed, converts it to rgb, in order to do some processing.

It works fine on the old camera implementation which uses NV21 Y

Answer 1

As you have not tagged this question as java or ndk. My replie implementation is from ndk or c++ point of view:

I know you want to convert the AIMAGE_FORMAT_YUV_420_888 to RGB b bits. Instead you can get directly AIMAGE_FORMAT_RGB_888 format if you request for AIMAGE_FORMAT during call to AImageReader_new().

/* 
 * Callback when image is available for processing 
 */
static void imageCallback(void* context, AImageReader* reader)
{
    AImage *image = nullptr;
    auto status = AImageReader_acquireNextImage(reader, &image);
    // Check status here ...

    // Try to process data without blocking the callback
    std::thread processor([=](){

        uint8_t *data = nullptr;
        int len = 0;
        AImage_getPlaneData(image, 0, &data, &len);

        // Process data here
        // ...

        AImage_delete(image);
    });
    processor.detach();
}

/*
 * Create RGB reader 
 */
AImageReader* createRGBReader()
{
    AImageReader* reader = nullptr;
    media_status_t status = AImageReader_new(640, 480, AIMAGE_FORMAT_RGB_888,
                     4, &reader);

    //if (status != AMEDIA_OK)
        // Handle errors here

    AImageReader_ImageListener listener{
            .context = nullptr,
            .onImageAvailable = imageCallback,
    };

    AImageReader_setImageListener(reader, &listener);

    return reader;
}

/* 
 * Create the surface for this reader
 */
ANativeWindow* createSurface(AImageReader* reader) 
{
    ANativeWindow *nativeWindow;
    AImageReader_getWindow(reader, &nativeWindow);

    return nativeWindow;
}


...

createSession()
{
    ...
    createSurface(createRGBReader());
    ...
}

Answer 2

In my approach I use OpenCV Mat and script from https://gist.github.com/camdenfullmer/dfd83dfb0973663a7974

First of all you convert your YUV_420_888 Image to Mat with the code in the link above.

*mImage is my Image object which i get in ImageReader.OnImageAvailableListener

Mat mYuvMat = imageToMat(mImage);

public static Mat imageToMat(Image image) {
    ByteBuffer buffer;
    int rowStride;
    int pixelStride;
    int width = image.getWidth();
    int height = image.getHeight();
    int offset = 0;

    Image.Plane[] planes = image.getPlanes();
    byte[] data = new byte[image.getWidth() * image.getHeight() * ImageFormat.getBitsPerPixel(ImageFormat.YUV_420_888) / 8];
    byte[] rowData = new byte[planes[0].getRowStride()];

    for (int i = 0; i < planes.length; i++) {
        buffer = planes[i].getBuffer();
        rowStride = planes[i].getRowStride();
        pixelStride = planes[i].getPixelStride();
        int w = (i == 0) ? width : width / 2;
        int h = (i == 0) ? height : height / 2;
        for (int row = 0; row < h; row++) {
            int bytesPerPixel = ImageFormat.getBitsPerPixel(ImageFormat.YUV_420_888) / 8;
            if (pixelStride == bytesPerPixel) {
                int length = w * bytesPerPixel;
                buffer.get(data, offset, length);

                if (h - row != 1) {
                    buffer.position(buffer.position() + rowStride - length);
                }
                offset += length;
            } else {


                if (h - row == 1) {
                    buffer.get(rowData, 0, width - pixelStride + 1);
                } else {
                    buffer.get(rowData, 0, rowStride);
                }

                for (int col = 0; col < w; col++) {
                    data[offset++] = rowData[col * pixelStride];
                }
            }
        }
    }

    Mat mat = new Mat(height + height / 2, width, CvType.CV_8UC1);
    mat.put(0, 0, data);

    return mat;
}

We have 1 channel YUV Mat. Define new Mat for BGR(not RGB yet) image:

Mat bgrMat = new Mat(mImage.getHeight(), mImage.getWidth(),CvType.CV_8UC4);

I just started learning OpenCV so propably this doesn't have to be 4-channel Mat and instead could be 3-channel but it works for me. Now I use convert color method to change my yuv Mat into bgr Mat.

Imgproc.cvtColor(mYuvMat, bgrMat, Imgproc.COLOR_YUV2BGR_I420);

Now we can do all the image processing like finding contours, colors, circles, etc. To print image back on screen we need to convert it to bitmap:

Mat rgbaMatOut = new Mat();
Imgproc.cvtColor(bgrMat, rgbaMatOut, Imgproc.COLOR_BGR2RGBA, 0);
final Bitmap bitmap = Bitmap.createBitmap(bgrMat.cols(), bgrMat.rows(), Bitmap.Config.ARGB_8888);
Utils.matToBitmap(rgbaMatOut, bitmap);

I have all my image processing in seperate thread so to set my ImageView I need to do this on the UI thread.

runOnUiThread(new Runnable() {
                    @Override
                    public void run() {
                        if(bitmap != null) {
                            mImageView.setImageBitmap(bitmap);
                        }
                    }
                });

Answer 3

Use Shyam Kumar's answer is not right for my phone, but Daniel Więcek's is right.I debug it, find planes[i].getRowStride() is 1216, planes[i].getPixelStride() is 2. While image width and height is both 1200.

Because my reputation is 3, so I cann't comment but post an answer.

Answer 4

Have you tried using this script? It's an answer posted by yydcdut on this question

https://github.com/pinguo-yuyidong/Camera2/blob/master/camera2/src/main/rs/yuv2rgb.rs

Answer 5

Approximately 10 times faster than the mentioned "imageToMat"-Function above is this code:

Image image = reader.acquireLatestImage();
...
Mat yuv = new Mat(image.getHeight() + image.getHeight() / 2, image.getWidth(), CvType.CV_8UC1);
ByteBuffer buffer = image.getPlanes()[0].getBuffer();
final byte[] data = new byte[buffer.limit()];
buffer.get(data);
yuv.put(0, 0, data);
...
image.close();

Answer 6

Camera2 YUV_420_888 to RGB Mat(opencv) in Java

@Override
    public void onImageAvailable(ImageReader reader){
        Image image = null;

        try {
            image = reader.acquireLatestImage();
            if (image != null) {

                byte[] nv21;
                ByteBuffer yBuffer = mImage.getPlanes()[0].getBuffer();
                ByteBuffer uBuffer = mImage.getPlanes()[1].getBuffer();
                ByteBuffer vBuffer = mImage.getPlanes()[2].getBuffer();

                int ySize = yBuffer.remaining();
                int uSize = uBuffer.remaining();
                int vSize = vBuffer.remaining();

                nv21 = new byte[ySize + uSize + vSize];

                //U and V are swapped
                yBuffer.get(nv21, 0, ySize);
                vBuffer.get(nv21, ySize, vSize);
                uBuffer.get(nv21, ySize + vSize, uSize);

                Mat mRGB = getYUV2Mat(nv21);



            }
        } catch (Exception e) {
            Log.w(TAG, e.getMessage());
        }finally{
            image.close();// don't forget to close
        }
    }



    public Mat getYUV2Mat(byte[] data) {
    Mat mYuv = new Mat(image.getHeight() + image.getHeight() / 2, image.getWidth(), CV_8UC1);
    mYuv.put(0, 0, data);
    Mat mRGB = new Mat();
    cvtColor(mYuv, mRGB, Imgproc.COLOR_YUV2RGB_NV21, 3);
    return mRGB;
}