问题
I keep hoping some code will appear on the internet, but getting nowhere ;)
WebRTC incoming I420Frame object seems to have 3 arrays of yuvPlanes
A typical Android camera app gets PreviewCallback.onPreviewFrame byte[] as a single array of bytes.
Can someone help me in how to convert I420Frames yuvPlanes to a single byte[] array like PreviewCallback.onPreviewFrame byte[] YCbCr_420_SP (NV21)?
For reference, VideoStreamsView.java has this code to render to OpenGL - but I just want it like camera preview ;) From: https://code.google.com/p/libjingle/source/browse/trunk/talk/examples/android/src/org/appspot/apprtc/VideoStreamsView.java?r=286
// Upload the YUV planes from |frame| to |textures|.
private void texImage2D(I420Frame frame, int[] textures) {
for (int i = 0; i < 3; ++i) {
ByteBuffer plane = frame.yuvPlanes[i];
GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textures[i]);
int w = i == 0 ? frame.width : frame.width / 2;
int h = i == 0 ? frame.height : frame.height / 2;
abortUnless(w == frame.yuvStrides[i], frame.yuvStrides[i] + "!=" + w);
GLES20.glTexImage2D(
GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, w, h, 0,
GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, plane);
}
checkNoGLES2Error();
}
Thank you.
回答1:
OK, here you go:
// Copy the bytes out of |src| and into |dst|, ignoring and overwriting
// positon & limit in both buffers.
//** copied from org/webrtc/VideoRenderer.java **//
private static void copyPlane(ByteBuffer src, ByteBuffer dst) {
src.position(0).limit(src.capacity());
dst.put(src);
dst.position(0).limit(dst.capacity());
}
public static android.graphics.YuvImage ConvertTo(org.webrtc.VideoRenderer.I420Frame src, int imageFormat) {
switch (imageFormat) {
default:
return null;
case android.graphics.ImageFormat.YV12: {
byte[] bytes = new byte[src.yuvStrides[0]*src.height +
src.yuvStrides[1]*src.height/2 +
src.yuvStrides[2]*src.height/2];
ByteBuffer tmp = ByteBuffer.wrap(bytes, 0, src.yuvStrides[0]*src.height);
copyPlane(src.yuvPlanes[0], tmp);
tmp = ByteBuffer.wrap(bytes, src.yuvStrides[0]*src.height, src.yuvStrides[2]*src.height/2);
copyPlane(src.yuvPlanes[2], tmp);
tmp = ByteBuffer.wrap(bytes, src.yuvStrides[0]*src.height+src.yuvStrides[2]*src.height/2, src.yuvStrides[1]*src.height/2);
copyPlane(src.yuvPlanes[1], tmp);
int[] strides = src.yuvStrides.clone();
return new YuvImage(bytes, imageFormat, src.width, src.height, strides);
}
case android.graphics.ImageFormat.NV21: {
if (src.yuvStrides[0] != src.width)
return convertLineByLine(src);
if (src.yuvStrides[1] != src.width/2)
return convertLineByLine(src);
if (src.yuvStrides[2] != src.width/2)
return convertLineByLine(src);
byte[] bytes = new byte[src.yuvStrides[0]*src.height +
src.yuvStrides[1]*src.height/2 +
src.yuvStrides[2]*src.height/2];
ByteBuffer tmp = ByteBuffer.wrap(bytes, 0, src.width*src.height);
copyPlane(src.yuvPlanes[0], tmp);
byte[] tmparray = new byte[src.width/2*src.height/2];
tmp = ByteBuffer.wrap(tmparray, 0, src.width/2*src.height/2);
copyPlane(src.yuvPlanes[2], tmp);
for (int row=0; row<src.height/2; row++) {
for (int col=0; col<src.width/2; col++) {
bytes[src.width*src.height + row*src.width + col*2] = tmparray[row*src.width/2 + col];
}
}
copyPlane(src.yuvPlanes[1], tmp);
for (int row=0; row<src.height/2; row++) {
for (int col=0; col<src.width/2; col++) {
bytes[src.width*src.height + row*src.width + col*2+1] = tmparray[row*src.width/2 + col];
}
}
return new YuvImage(bytes, imageFormat, src.width, src.height, null);
}
}
}
public static android.graphics.YuvImage convertLineByLine(org.webrtc.VideoRenderer.I420Frame src) {
byte[] bytes = new byte[src.width*src.height*3/2];
int i=0;
for (int row=0; row<src.height; row++) {
for (int col=0; col<src.width; col++) {
bytes[i++] = src.yuvPlanes[0][col+row*src.yuvStrides[0]];
}
}
for (int row=0; row<src.height/2; row++) {
for (int col=0; col<src.width/2; col++) {
bytes[i++] = src.yuvPlanes[2][col+row*src.yuvStrides[2]];
bytes[i++] = src.yuvPlanes[1][col+row*src.yuvStrides[1]];
}
}
return new YuvImage(bytes, android.graphics.ImageFormat.NV21, src.width, src.height, null);
}
}
This converts I420Frame to Android YuvImage of android.graphics.ImageFormat.NV21, which you can compressToJpeg(). ImageFormat.YV12 support seems to be limited in SDK. Note that the Y and V must be shuffled.
Most error checking is skipped for brevity.
回答2:
While the question is pretty old, I'm posting this answer for anyone who need to do this conversion. I found the below code in one of the WebRTC test cases, and it works perfectly.
private void testI420toNV21Conversion(ImageView iv) {
// create a test I420 frame. Instead of this you can use the actual frame
VideoFrame.I420Buffer i420Buffer = createTestI420Buffer();
final int width = i420Buffer.getWidth();
final int height = i420Buffer.getHeight();
//convert to nv21, this is the same as byte[] from onPreviewCallback
byte[] nv21Data = createNV21Data(i420Buffer);
//let's test the conversion by converting the NV21 data to jpg and showing it in a bitmap.
YuvImage yuvImage = new YuvImage(nv21Data, ImageFormat.NV21,width,height,null);
ByteArrayOutputStream out = new ByteArrayOutputStream();
yuvImage.compressToJpeg(new Rect(0, 0, width, height), 100, out);
byte[] imageBytes = out.toByteArray();
Bitmap image = BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.length);
iv.setImageBitmap(image);
}
.
/** Create an NV21Buffer with the same pixel content as the given I420 buffer. */
public static byte[] createNV21Data(VideoFrame.I420Buffer i420Buffer) {
final int width = i420Buffer.getWidth();
final int height = i420Buffer.getHeight();
final int chromaStride = width;
final int chromaWidth = (width + 1) / 2;
final int chromaHeight = (height + 1) / 2;
final int ySize = width * height;
final ByteBuffer nv21Buffer = ByteBuffer.allocateDirect(ySize + chromaStride * chromaHeight);
// We don't care what the array offset is since we only want an array that is direct.
@SuppressWarnings("ByteBufferBackingArray") final byte[] nv21Data = nv21Buffer.array();
for (int y = 0; y < height; ++y) {
for (int x = 0; x < width; ++x) {
final byte yValue = i420Buffer.getDataY().get(y * i420Buffer.getStrideY() + x);
nv21Data[y * width + x] = yValue;
}
}
for (int y = 0; y < chromaHeight; ++y) {
for (int x = 0; x < chromaWidth; ++x) {
final byte uValue = i420Buffer.getDataU().get(y * i420Buffer.getStrideU() + x);
final byte vValue = i420Buffer.getDataV().get(y * i420Buffer.getStrideV() + x);
nv21Data[ySize + y * chromaStride + 2 * x + 0] = vValue;
nv21Data[ySize + y * chromaStride + 2 * x + 1] = uValue;
}
}
return nv21Data;
}
/** Convert a byte array to a direct ByteBuffer. */
private static ByteBuffer toByteBuffer(int[] array) {
final ByteBuffer buffer = ByteBuffer.allocateDirect(array.length);
buffer.put(toByteArray(array));
buffer.rewind();
return buffer;
}
/**
* Convert an int array to a byte array and make sure the values are within the range [0, 255].
*/
private static byte[] toByteArray(int[] array) {
final byte[] res = new byte[array.length];
for (int i = 0; i < array.length; ++i) {
final int value = array[i];
res[i] = (byte) value;
}
return res;
}
public static VideoFrame.I420Buffer createTestI420Buffer() {
final int width = 16;
final int height = 16;
final int[] yData = new int[] {156, 162, 167, 172, 177, 182, 187, 193, 199, 203, 209, 214, 219,
224, 229, 235, 147, 152, 157, 162, 168, 173, 178, 183, 188, 193, 199, 205, 210, 215, 220,
225, 138, 143, 148, 153, 158, 163, 168, 174, 180, 184, 190, 195, 200, 205, 211, 216, 128,
133, 138, 144, 149, 154, 159, 165, 170, 175, 181, 186, 191, 196, 201, 206, 119, 124, 129,
134, 140, 145, 150, 156, 161, 166, 171, 176, 181, 187, 192, 197, 109, 114, 119, 126, 130,
136, 141, 146, 151, 156, 162, 167, 172, 177, 182, 187, 101, 105, 111, 116, 121, 126, 132,
137, 142, 147, 152, 157, 162, 168, 173, 178, 90, 96, 101, 107, 112, 117, 122, 127, 132, 138,
143, 148, 153, 158, 163, 168, 82, 87, 92, 97, 102, 107, 113, 118, 123, 128, 133, 138, 144,
149, 154, 159, 72, 77, 83, 88, 93, 98, 103, 108, 113, 119, 124, 129, 134, 139, 144, 150, 63,
68, 73, 78, 83, 89, 94, 99, 104, 109, 114, 119, 125, 130, 135, 140, 53, 58, 64, 69, 74, 79,
84, 89, 95, 100, 105, 110, 115, 120, 126, 131, 44, 49, 54, 59, 64, 70, 75, 80, 85, 90, 95,
101, 106, 111, 116, 121, 34, 40, 45, 50, 55, 60, 65, 71, 76, 81, 86, 91, 96, 101, 107, 113,
25, 30, 35, 40, 46, 51, 56, 61, 66, 71, 77, 82, 87, 92, 98, 103, 16, 21, 26, 31, 36, 41, 46,
52, 57, 62, 67, 72, 77, 83, 89, 94};
final int[] uData = new int[] {110, 113, 116, 118, 120, 123, 125, 128, 113, 116, 118, 120, 123,
125, 128, 130, 116, 118, 120, 123, 125, 128, 130, 132, 118, 120, 123, 125, 128, 130, 132,
135, 120, 123, 125, 128, 130, 132, 135, 138, 123, 125, 128, 130, 132, 135, 138, 139, 125,
128, 130, 132, 135, 138, 139, 142, 128, 130, 132, 135, 138, 139, 142, 145};
final int[] vData = new int[] {31, 45, 59, 73, 87, 100, 114, 127, 45, 59, 73, 87, 100, 114, 128,
141, 59, 73, 87, 100, 114, 127, 141, 155, 73, 87, 100, 114, 127, 141, 155, 168, 87, 100,
114, 128, 141, 155, 168, 182, 100, 114, 128, 141, 155, 168, 182, 197, 114, 127, 141, 155,
168, 182, 196, 210, 127, 141, 155, 168, 182, 196, 210, 224};
return JavaI420Buffer.wrap(width, height, toByteBuffer(yData),
/* strideY= */ width, toByteBuffer(uData), /* strideU= */ width / 2, toByteBuffer(vData),
/* strideV= */ width / 2,
/* releaseCallback= */ null);
}
来源：https://stackoverflow.com/questions/20803423/android-org-webrtc-videorenderer-i420frame-arrays-to-previewcallback-onpreviewfr