Android: Run image classifier in infinite loop

问题

I want to run a image classifier in a infinite loop on a background thread. The function should be called immediately after launching the app. I want to feed the classifier with current frames from a prerecorded video which is simultaneously playing in the UI-thread, so the background thread should tell the UI-thread, once it's done, so I can feed it with the current frame and rerun the classifier.

public class MainActivity extends AppCompatActivity {
    private static final String TAG = "MainActivity";

    private VideoView videoView;
    private ImageView imageView;
    private Uri uri_video;
    private MediaMetadataRetriever mediaMetadataRetriever = new MediaMetadataRetriever();
    private MediaController mMediaController;

    private static volatile int currentPosition;
    private static volatile Bitmap mBitmap;

    private final Object lock = new Object();
    private volatile boolean runClassifier = false;

    private HandlerThread backgroundThread;
    private Handler backgroundHandler;
    private static final String HANDLE_THREAD_NAME = "ClassifierBackground";


    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);

        imageView = findViewById(R.id.imageView);
        uri_video = Uri.parse("android.resource://" + getPackageName() + "/" + R.raw.kim);
        mediaMetadataRetriever = new MediaMetadataRetriever();
        mediaMetadataRetriever.setDataSource(getApplication(), uri_video);
        videoView = findViewById(R.id.videoView);
        videoView.setVideoURI(uri_video);
        mMediaController = new MediaController(this);
        videoView.setMediaController(mMediaController);
        videoView.setOnPreparedListener(MyVideoViewPreparedListener);
        videoView.start();

        startBackgroundThread();
    }


    /** Starts a background thread and its {@link Handler}. */
    private void startBackgroundThread() {
        backgroundThread = new HandlerThread(HANDLE_THREAD_NAME);
        backgroundThread.start();
        backgroundHandler = new Handler(backgroundThread.getLooper());
        synchronized (lock) {
            runClassifier = true;
        }
        backgroundHandler.post(periodicClassify);
    }

    /** Stops the background thread and its {@link Handler}. */
    private void stopBackgroundThread() {
        backgroundThread.quitSafely();
        try {
            backgroundThread.join();
            backgroundThread = null;
            backgroundHandler = null;
            synchronized (lock) {
                runClassifier = false;
            }
        } catch (InterruptedException e) {
            Log.e(TAG, "Interrupted when stopping background thread", e);
        }
    }

    private Runnable periodicClassify =
            new Runnable() {
                @Override
                public void run() {
                    synchronized (lock) {
                        if (runClassifier) {
                            //  classifyFrame(); // This will be implemented later
                            Log.d(TAG, "run: Classifier is running");
                            SystemClock.sleep(100); // Instead I simulate the classifier via sleep
                        }
                    }
                    runOnUiThread(new Runnable() {
                        @Override
                        public void run() {
                            setImageViewToCurrentFrame();
                        }
                    });
                    backgroundHandler.post(periodicClassify);
                }
            };


    private void setImageViewToCurrentFrame(){
        currentPosition = videoView.getCurrentPosition(); //in millisecond
        mBitmap = mediaMetadataRetriever
                .getFrameAtTime(currentPosition * 1000); //unit in microsecond
        imageView.setImageBitmap(mBitmap);
    }


    MediaPlayer.OnPreparedListener MyVideoViewPreparedListener =
            new MediaPlayer.OnPreparedListener() {

                @Override
                public void onPrepared(MediaPlayer mp) {

                    long duration = videoView.getDuration(); //in millisecond
                    Toast.makeText(MainActivity.this,
                            "Duration: " + duration + " (ms)",
                            Toast.LENGTH_LONG).show();
                    setImageViewToCurrentFrame();
                }
            };


    @Override
    protected void onDestroy() {
        super.onDestroy();
        stopBackgroundThread();
    }

EDIT1:
I got some rough idea on how to do it from these videos. It seems like I need a backgroundThread (HandlerThread) that has a backgroundHandler (Handler) to communicate with the UI-thread and a Looper to keep the background thread alive. setImageViewToCurrentFrame uses videoView.getCurrentPosition() to update the mBitmap.

However the update is very slow (>10 seconds) compared to the runtime of the classifier (SystemClock.sleep(100) which takes 100ms).

EDIT2: The problem seems to be the performance of ImageView which seems to be updated very slowly. Replacing it with TextView, keeps both the background thread and the UI-thread in sync. I will look for other solutions than ImageView now

回答1:

Here is the gist of my solution. One needs to fit in the actual image classifier at the SystemClock.sleep part. The trick is to use TextureView instead of ImageView or VideoView, since it's faster and more flexible

@Override
protected void onCreate(Bundle savedInstanceState) {
    super.onCreate(savedInstanceState);
    setContentView(R.layout.video_playback);
    initializeBottomSheet();


    uri_video = Uri.parse("android.resource://" + getPackageName() + "/" + R.raw.my_demo_video);

    textureView = findViewById(R.id.textureView);
    textureView.setSurfaceTextureListener(this);

    mediaPlayer = new MediaPlayer();
    assert textureView != null;

    startBackgroundThread();
}




/** Starts a background thread and its {@link Handler}. */
private void startBackgroundThread() {
    backgroundThread = new HandlerThread(HANDLE_THREAD_NAME);
    backgroundThread.start();
    backgroundHandler = new Handler(backgroundThread.getLooper());
    synchronized (lock) {
        runClassifier = true;
    }
    backgroundHandler.post(periodicClassify);
}

/** Stops the background thread and its {@link Handler}. */
private void stopBackgroundThread() {
    backgroundThread.quitSafely();
    try {
        backgroundThread.join();
        backgroundThread = null;
        backgroundHandler = null;
        synchronized (lock) {
            runClassifier = false;
        }
    } catch (InterruptedException e) {
        Log.e(TAG, "Interrupted when stopping background thread", e);
    }
}

private Runnable periodicClassify =
        new Runnable() {
            @Override
            public void run() {
                // Get current frame from video playback
                mBitmap = textureView.getBitmap();
                if (classifier != null && mBitmap != null) {
                    Log.d(TAG, "Classifier: Start thread");
                    SystemClock.sleep(3000); // Instead I simulate the classifier via sleep
                }
                backgroundHandler.post(periodicClassify);
            }
        };


@Override
public void onSurfaceTextureAvailable(SurfaceTexture surfaceTexture, int width, int height) {
    Surface surface = new Surface(surfaceTexture);
    Context context = getApplicationContext();
    try {
        mediaPlayer.setDataSource(context, uri_video);
        mediaPlayer.setSurface(surface);
        mediaPlayer.prepareAsync();
        mediaPlayer.setOnPreparedListener(new MediaPlayer.OnPreparedListener(){

            @Override
            public void onPrepared(MediaPlayer mediaPlayer) {
                mediaPlayer.start();

                textureView.setOnClickListener(new View.OnClickListener() {
                    @Override
                    public void onClick(View view) {
                        if (mediaPlayer.isPlaying()){
                            mediaPlayer.pause();
                        }else{
                            mediaPlayer.start();
                        }
                    }
                });
            }
        });
    } catch (IOException e) {
        e.printStackTrace();
    }



}



@Override
protected void onDestroy() {
    super.onDestroy();
    stopBackgroundThread();
    if(mediaPlayer != null){
        mediaPlayer.stop();
        mediaPlayer.release();
        mediaPlayer = null;
    }
}

@Override
protected void onPause() {
    super.onPause();
    if(mediaPlayer != null && mediaPlayer.isPlaying()){
        mediaPlayer.pause();
    }
}

@Override
protected void onResume() {
    super.onResume();
    if(mediaPlayer != null && mediaPlayer.isPlaying()){
        mediaPlayer.start();
    }
    startBackgroundThread();
}

来源：https://stackoverflow.com/questions/57788197/android-run-image-classifier-in-infinite-loop