Streaming CustomView ARcore with Twilio video

Question

I\'ve a problem when i want to streaming a custom view with twilio video api along with ARcore, basically it stream a black screen. I used ViewCapturer class from example to

Answer 1

For someone who must do stream ARCore with Twilio Video

In your ARCore render class.

@Override
public void onDrawFrame(GL10 gl) {
     ....
     this.takeLastFrame();
}

private byte[] takeLastFrame() {
    int height = this.mFrameHeight;
    int width = this.mFrameWidth;

    Mat input = new Mat(height, width, CvType.CV_8UC4);

    ByteBuffer buffer = ByteBuffer.allocate(input.rows() * input.cols() * input.channels());

    GLES20.glReadPixels(0, 0, width, height,
            GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, buffer);
    input.put(0, 0, buffer.array());

    Core.rotate(input, input, Core.ROTATE_180);
    Core.flip(input, input, 1);

    return convertMatToBytes(input);
}

private byte[] convertMatToBytes(Mat image) {
    int bufferSize = image.channels() * image.cols() * image.rows();
    byte[] b = new byte[bufferSize];
    image.get(0, 0, b);
    return b;
}

In your custom capturer class

byte[] array = view.takeLastFrame();
if (array != null && array.length > 0) {
    final long captureTimeNs = TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime());

    // Create video frame
    VideoDimensions dimensions = new VideoDimensions(view.getFrameWidth(), view.getFrameHeight());
    VideoFrame videoFrame = new VideoFrame(array,
                        dimensions, VideoFrame.RotationAngle.ROTATION_0, captureTimeNs);

    // Notify the listener
    if (started.get()) {
        videoCapturerListener.onFrameCaptured(videoFrame);
    }
}