Mediacodec, decode byte packet from server and render it on surface

问题

I have some issues with MediaCode.

I have 3 components; Decoder, Downloader and Render. And Simple FragmentStreamVideo that initialize the 'SurfaceView' and the 'Downloader'.

The other components like the Render and Decoder are initialized in the SurfaceView. Then, a syncronize is done between the Decoder and the Dowloader, implemented by BlockingQueue<String> queue where String = Filename (Each frame has its file).

Another syncronize between Decode and Render is done by the standard ByteBuffer as stated in documentation.

Find below my piece of code. Would be very grateful if you can help.

SurfaceView

public class MySurfaceView extends SurfaceView implements SurfaceHolder.Callback {
private Render drawThread;
private Decoder decoder;
BlockingQueue<String> queue;
Context mContext;
public MySurfaceView(Context context,BlockingQueue<String> queue){
    super(context);
    this.queue = queue;
    mContext = context;
    getHolder().addCallback(this);
}
public void setRunning(boolean value){
    drawThread.setRunning(value);
}

@Override
public void surfaceCreated(SurfaceHolder holder) {
    //drawThread = new Render(getHolder().getSurface(),300,300);
    drawThread = new Render(getHolder().getSurface(),352,288);
    decoder = new Decoder(drawThread,queue,mContext);
    decoder.start();
    drawThread.setRunning(true);
    drawThread.start();
}

@Override
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {

}

@Override
public void surfaceDestroyed(SurfaceHolder holder) {
    boolean retry = true;
    // завершаем работу потока
    drawThread.setRunning(false);
    while (retry) {
        try {
            drawThread.join();
            retry = false;
        } catch (InterruptedException e) {
            // если не получилось, то будем пытаться еще и еще
        }
    }

}

Render

public class Render extends Thread{
private boolean mConfigured = false;
private long mTimeoutUs;
private MediaCodec mDecoder;
volatile boolean mRunning = false;
byte[] header_sps = { 0, 0, 0, 1, 103, 100, 0, 40, -84, 52, -59, 1, -32, 17, 31, 120, 11, 80, 16, 16, 31, 0, 0, 3, 3, -23, 0, 0, -22, 96, -108 };
byte[] header_pps = { 0, 0, 0, 1, 104, -18, 60, -128 };

@TargetApi(Build.VERSION_CODES.JELLY_BEAN)
public Render(Surface surface, int width, int height){
    mTimeoutUs = 1000l;
    //возможны проблемы с инициализацией
    ByteBuffer csd0 = ByteBuffer.allocate(2);
    csd0.put((byte) MediaCodec.BUFFER_FLAG_CODEC_CONFIG);

    try {
        configure(surface,width,height,csd0);
    } catch (IOException e) {
        e.printStackTrace();
    }
}

synchronized void configure(Surface surface, int width, int height,
                            ByteBuffer csd0) throws IOException {
    if (mConfigured) { // просто флаг, чтобы знать, что декодер готов
        throw new IllegalStateException();
    }
    // создаем видео формат
    MediaFormat format = MediaFormat.createVideoFormat("video/avc", width, height);
    // передаем наш csd-0
    //format.setByteBuffer("csd-0", csd0);
    //format.setByteBuffer("csd-0",ByteBuffer.wrap(header_sps));
    format.setByteBuffer("csd-1", ByteBuffer.wrap(header_pps));
    // создаем декодер
    mDecoder = MediaCodec.createDecoderByType("video/avc");
    // конфигурируем декодер
    mDecoder.configure(format, surface, null, 0);
    mDecoder.start();
    mConfigured = true;
}
void decodeSample(byte[] data, int offset, int size, long presentationTimeUs, int flags) {
    if (mConfigured) {
        // вызов блокирующий
        int index = mDecoder.dequeueInputBuffer(mTimeoutUs);
        if (index >= 0) {
            ByteBuffer buffer = mDecoder.getInputBuffers()[index];
            buffer.clear(); // обязательно сбросить позицию и размер буфера
            buffer.put(data, offset, size);
            // сообщаем системе о доступности буфера данных
            mDecoder.queueInputBuffer(index, 0, size, presentationTimeUs, flags);
        }
    }
}
@TargetApi(Build.VERSION_CODES.JELLY_BEAN)
@Override
public void run() {
    try {
        MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); // переиспользуем BufferInfo
        long startMs = System.currentTimeMillis();
        while (mRunning) {
            if (mConfigured) { // если кодек готов
                int index = mDecoder.dequeueOutputBuffer(info, mTimeoutUs);
                if (index >= 0) { // буфер с индексом index доступен
                    // info.size > 0: если буфер не нулевого размера, то рендерим на Surface
                    while (info.presentationTimeUs / 1000 > System.currentTimeMillis() - startMs) {
                        try {
                            sleep(10);
                        } catch (InterruptedException e) {
                            e.printStackTrace();
                            break;
                        }
                    }
                    mDecoder.releaseOutputBuffer(index, info.size > 0);
                    // заканчиваем работу декодера если достигнут конец потока данных
                    if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == MediaCodec.BUFFER_FLAG_END_OF_STREAM) {
                        mRunning = false;
                        break;
                    }
                }
            } else {
                // просто спим, т.к. кодек не готов
                try {
                    Thread.sleep(10);
                } catch (InterruptedException ignore) {
                }
            }
        }
    } finally {
        // освобождение ресурсов
        release();
    }
}
public void setRunning(boolean value){
    mRunning = value;
}
void release() {
    if (mConfigured) {
        mDecoder.stop();
        mDecoder.release();
    }
}

Decoder

public class Render extends Thread{
private boolean mConfigured = false;
private long mTimeoutUs;
private MediaCodec mDecoder;
volatile boolean mRunning = false;
byte[] header_sps = { 0, 0, 0, 1, 103, 100, 0, 40, -84, 52, -59, 1, -32, 17, 31, 120, 11, 80, 16, 16, 31, 0, 0, 3, 3, -23, 0, 0, -22, 96, -108 };
byte[] header_pps = { 0, 0, 0, 1, 104, -18, 60, -128 };

@TargetApi(Build.VERSION_CODES.JELLY_BEAN)
public Render(Surface surface, int width, int height){
    mTimeoutUs = 1000l;
    //возможны проблемы с инициализацией
    ByteBuffer csd0 = ByteBuffer.allocate(2);
    csd0.put((byte) MediaCodec.BUFFER_FLAG_CODEC_CONFIG);

    try {
        configure(surface,width,height,csd0);
    } catch (IOException e) {
        e.printStackTrace();
    }
}

synchronized void configure(Surface surface, int width, int height,
                            ByteBuffer csd0) throws IOException {
    if (mConfigured) { // просто флаг, чтобы знать, что декодер готов
        throw new IllegalStateException();
    }
    // создаем видео формат
    MediaFormat format = MediaFormat.createVideoFormat("video/avc", width, height);
    // передаем наш csd-0
    //format.setByteBuffer("csd-0", csd0);
    //format.setByteBuffer("csd-0",ByteBuffer.wrap(header_sps));
    format.setByteBuffer("csd-1", ByteBuffer.wrap(header_pps));
    // создаем декодер
    mDecoder = MediaCodec.createDecoderByType("video/avc");
    // конфигурируем декодер
    mDecoder.configure(format, surface, null, 0);
    mDecoder.start();
    mConfigured = true;
}
void decodeSample(byte[] data, int offset, int size, long presentationTimeUs, int flags) {
    if (mConfigured) {
        // вызов блокирующий
        int index = mDecoder.dequeueInputBuffer(mTimeoutUs);
        if (index >= 0) {
            ByteBuffer buffer = mDecoder.getInputBuffers()[index];
            buffer.clear(); // обязательно сбросить позицию и размер буфера
            buffer.put(data, offset, size);
            // сообщаем системе о доступности буфера данных
            mDecoder.queueInputBuffer(index, 0, size, presentationTimeUs, flags);
        }
    }
}
@TargetApi(Build.VERSION_CODES.JELLY_BEAN)
@Override
public void run() {
    try {
        MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); // переиспользуем BufferInfo
        long startMs = System.currentTimeMillis();
        while (mRunning) {
            if (mConfigured) { // если кодек готов
                int index = mDecoder.dequeueOutputBuffer(info, mTimeoutUs);
                if (index >= 0) { // буфер с индексом index доступен
                    // info.size > 0: если буфер не нулевого размера, то рендерим на Surface
                    while (info.presentationTimeUs / 1000 > System.currentTimeMillis() - startMs) {
                        try {
                            sleep(10);
                        } catch (InterruptedException e) {
                            e.printStackTrace();
                            break;
                        }
                    }
                    mDecoder.releaseOutputBuffer(index, info.size > 0);
                    // заканчиваем работу декодера если достигнут конец потока данных
                    if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == MediaCodec.BUFFER_FLAG_END_OF_STREAM) {
                        mRunning = false;
                        break;
                    }
                }
            } else {
                // просто спим, т.к. кодек не готов
                try {
                    Thread.sleep(10);
                } catch (InterruptedException ignore) {
                }
            }
        }
    } finally {
        // освобождение ресурсов
        release();
    }
}
public void setRunning(boolean value){
    mRunning = value;
}
void release() {
    if (mConfigured) {
        mDecoder.stop();
        mDecoder.release();
    }
}
}

Downloader (Not full because more function Work)
in readFramePacket function I'm save file and put it on queue it's work fine.

public class Downloader extends Thread{
private final byte COMMAND_ONE = (byte)0x01;
private final byte COMMAND_TEN = (byte)0x10;

private final byte MAIN_TYPE_STREAM =(byte) 0x00;
private final byte SUPPORT_TYPE_STREAM =(byte) 0x01;
private final byte CONTROL_CODE_DISCONNECT =(byte) 0x00;
private final byte CONTROL_CODE_CONNECT =(byte) 0x01;

private final byte DELIMITER = (byte)':';
private final int FIX_PACKET_HEAD_SIZE = 5;
private final int SECOND_HEAD_FIX_SIZE = 4;
private final int FIX_DELIMITER_SIZE = 1;
private static final int HEAD_OF_PACKET = -1;
private static final int SECOND_HEAD_OF_PACKET = 0;
private static final int DATA_PACKET = 1;
private static final int PACKETS_END = -2;
private static final int LAST_PACKET = 8;
private static final int COMMAND_INDEX = 0;
private static final int SOCKET_TIMEOUT_LENGTH = 5000;
private String log;
private String pass;
String host;
int portNumber;
BlockingQueue<String> queue;
Socket socket;
Context mContext;
OutputStream out;
int cameraID;
int value = 0;

volatile boolean mRunning = false;


public Downloader(Server server,BlockingQueue<String> queue,int cameraID,Context context){
    log = server.getLogin();
    pass = server.getPassword();
    host = server.getIP();
    portNumber = Integer.parseInt(server.getPort());
    this.queue = queue;
    mRunning = true;
    this.cameraID = cameraID;
    mContext = context;
}
public void stopStreaming(){
    mRunning = false;
}

@Override
public void run() {
    try {
        SocketAddress sockaddr = new InetSocketAddress(host, portNumber);
        socket = new Socket();
        socket.connect(sockaddr, SOCKET_TIMEOUT_LENGTH);
        //socket.setSoTimeout(SOCKET_TIMEOUT_LENGTH);

        InputStream inputStream = new BufferedInputStream(socket.getInputStream());
        out = socket.getOutputStream();
        byte[] command = createAuthenticationPacket(log, pass);
        out.write(command);
        readAuthPacket(inputStream);
        byte[] connectCommand = packetToConnectDisconnectCamera(cameraID,SUPPORT_TYPE_STREAM,CONTROL_CODE_CONNECT);
        out.write(connectCommand);
        initializeVideoStream(inputStream);
        readFramePacket(inputStream);

    } catch (SocketException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    } catch (NoSuchAlgorithmException e) {
        e.printStackTrace();
    } catch (NoSuchProviderException e) {
        e.printStackTrace();
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
}

来源：https://stackoverflow.com/questions/29897701/mediacodec-decode-byte-packet-from-server-and-render-it-on-surface