c++ fast screenshots in linux for use with opencv

Question

I\'m trying to find a fast way to take screenshots (as in 30 fps or over) for use with opencv in c++

All the information I\'ve found online either involved windows.h or

Answer 1

You can use this to get the screenshot into a structure of raw pixels. Pass that to OpenCV along with the Width & Height & BitsPerPixel and you should be good.

#include <X11/Xlib.h>
#include <X11/Xutil.h>
#include <cstdint>
#include <cstring>
#include <vector>

void ImageFromDisplay(std::vector<uint8_t>& Pixels, int& Width, int& Height, int& BitsPerPixel)
{
    Display* display = XOpenDisplay(nullptr);
    Window root = DefaultRootWindow(display);

    XWindowAttributes attributes = {0};
    XGetWindowAttributes(display, root, &attributes);

    Width = attributes.width;
    Height = attributes.height;

    XImage* img = XGetImage(display, root, 0, 0 , Width, Height, AllPlanes, ZPixmap);
    BitsPerPixel = img->bits_per_pixel;
    Pixels.resize(Width * Height * 4);

    memcpy(&Pixels[0], img->data, Pixels.size());

    XDestroyImage(img);
    XCloseDisplay(display);
}

Then to use it with OpenCV, you can do:

int main()
{
    int Width = 0;
    int Height = 0;
    int Bpp = 0;
    std::vector<std::uint8_t> Pixels;

    ImageFromDisplay(Pixels, Width, Height, Bpp);

    if (Width && Height)
    {
        Mat img = Mat(Height, Width, Bpp > 24 ? CV_8UC4 : CV_8UC3, &Pixels[0]); //Mat(Size(Height, Width), Bpp > 24 ? CV_8UC4 : CV_8UC3, &Pixels[0]); 

        namedWindow("WindowTitle", WINDOW_AUTOSIZE);
        imshow("Display window", img);

        waitKey(0);
    }
    return 0;
}

Answer 2

Based on @abc's answer I came up with the following, using MIT's shared memory extension for X.

@abc's example runs at 120-180 fps and uses ~40% of a Titan X (Maxwell). The following runs at 15000 fps and uses 80% of a Titan X. (Assuming you don't sync. If you do sync, then the framerate drops to @abc's framerate!)

Comment out the break; statement inside the inner loop too run it nonstop.

Version using @abc's class:

// g++ screena.cpp -o screena -lX11 -lXext -Ofast -mfpmath=both -march=native -m64 -funroll-loops -mavx2 `pkg-config opencv --cflags --libs` && ./screena

#include <X11/Xlib.h>
#include <X11/Xutil.h>

#include <X11/extensions/XShm.h>
#include <sys/ipc.h>
#include <sys/shm.h>

#include <opencv2/opencv.hpp>  // This includes most headers!

#include <time.h>
#define FPS(start) (CLOCKS_PER_SEC / (clock()-start))


struct ScreenShot{
    ScreenShot(uint x, uint y, uint width, uint height):
               x(x), y(y), width(width), height(height){

        display = XOpenDisplay(nullptr);
        root = DefaultRootWindow(display);

        XGetWindowAttributes(display, root, &window_attributes);
        screen = window_attributes.screen;
        ximg = XShmCreateImage(display, DefaultVisualOfScreen(screen), DefaultDepthOfScreen(screen), ZPixmap, NULL, &shminfo, width, height);

        shminfo.shmid = shmget(IPC_PRIVATE, ximg->bytes_per_line * ximg->height, IPC_CREAT|0777);
        shminfo.shmaddr = ximg->data = (char*)shmat(shminfo.shmid, 0, 0);
        shminfo.readOnly = False;
        if(shminfo.shmid < 0)
            puts("Fatal shminfo error!");;
        Status s1 = XShmAttach(display, &shminfo);
        printf("XShmAttach() %s\n", s1 ? "success!" : "failure!");

        init = true;
    }

    void operator() (cv::Mat& cv_img){
        if(init)
            init = false;

        XShmGetImage(display, root, ximg, 0, 0, 0x00ffffff);
        cv_img = cv::Mat(height, width, CV_8UC4, ximg->data);
    }

    ~ScreenShot(){
        if(!init)
            XDestroyImage(ximg);

        XShmDetach(display, &shminfo);
        shmdt(shminfo.shmaddr);
        XCloseDisplay(display);
    }

    Display* display;
    Window root;
    XWindowAttributes window_attributes;
    Screen* screen;
    XImage* ximg;
    XShmSegmentInfo shminfo;

    int x, y, width, height;

    bool init;
};


int main(){
    ScreenShot screen(0, 0, 1920, 1080);
    cv::Mat img;

    for(uint i;; ++i){
        double start = clock();

        screen(img);

        if(!(i & 0b111111))
            printf("fps %4.f  spf %.4f\n", FPS(start), 1 / FPS(start));
            break;

    }

    cv::imshow("img", img);
    cv::waitKey(0);
}

"Naked" version:

// g++ xshm2.c -o xshm2 -lX11 -lXext `$cv`-Ofast -mfpmath=both -march=native -m64 -funroll-loops -mavx2 && ./xshm2

#include <X11/Xlib.h>
#include <X11/Xutil.h>

#include <X11/extensions/XShm.h>
#include <sys/ipc.h>
#include <sys/shm.h>

#include <opencv2/opencv.hpp>  // This includes most headers!

#include <time.h>
#define FPS(start) (CLOCKS_PER_SEC / (clock()-start))

// Using one monitor DOESN'T improve performance! Querying a smaller subset of the screen DOES
const uint WIDTH  = 1920>>0;
const uint HEIGHT = 1080>>0;

// -------------------------------------------------------
int main(){
    Display* display = XOpenDisplay(NULL);
    Window root = DefaultRootWindow(display);  // Macro to return the root window! It's a simple uint32
    XWindowAttributes window_attributes;
    XGetWindowAttributes(display, root, &window_attributes);
    Screen* screen = window_attributes.screen;
    XShmSegmentInfo shminfo;
    XImage* ximg = XShmCreateImage(display, DefaultVisualOfScreen(screen), DefaultDepthOfScreen(screen), ZPixmap, NULL, &shminfo, WIDTH, HEIGHT);

    shminfo.shmid = shmget(IPC_PRIVATE, ximg->bytes_per_line * ximg->height, IPC_CREAT|0777);
    shminfo.shmaddr = ximg->data = (char*)shmat(shminfo.shmid, 0, 0);
    shminfo.readOnly = False;
    if(shminfo.shmid < 0)
        puts("Fatal shminfo error!");;
    Status s1 = XShmAttach(display, &shminfo);
    printf("XShmAttach() %s\n", s1 ? "success!" : "failure!");

    cv::Mat img;

    for(int i; ; i++){
        double start = clock();

        XShmGetImage(display, root, ximg, 0, 0, 0x00ffffff);
        img = cv::Mat(HEIGHT, WIDTH, CV_8UC4, ximg->data);

        if(!(i & 0b111111))
            printf("fps %4.f  spf %.4f\n", FPS(start), 1 / FPS(start));
        break;
    }

    cv::imshow("img", img);
    cv::waitKey(0);

    XShmDetach(display, &shminfo);
    XDestroyImage(ximg);
    shmdt(shminfo.shmaddr);
    XCloseDisplay(display);
    puts("Exit success!");
}

Answer 3

#include <X11/Xlib.h>
#include <X11/Xutil.h>
#include <opencv2/opencv.hpp>
#include <opencv2/highgui.hpp>
#include <cstdint>
#include <cstring>
#include <vector>

using namespace cv;

struct ScreenShot
{
    ScreenShot(int x, int y, int width, int height):
        x(x),
        y(y),
        width(width),
        height(height)
    {
        display = XOpenDisplay(nullptr);
        root = DefaultRootWindow(display);

        init = true;
    }

    void operator() (Mat& cvImg)
    {
        if(init == true)
            init = false;
        else
            XDestroyImage(img);

        img = XGetImage(display, root, x, y, width, height, AllPlanes, ZPixmap);

        cvImg = Mat(height, width, CV_8UC4, img->data);
    }

    ~ScreenShot()
    {
        if(init == false)
            XDestroyImage(img);

        XCloseDisplay(display);
    }

    Display* display;
    Window root;
    int x,y,width,height;
    XImage* img;

    bool init;
};

int main(int, char**)
{
    for(;;){
        ScreenShot screen(0,0,1366,768);

        Mat img;
        screen(img);

        imshow("img", img);
        if(waitKey(30) >= 0) break;
    }
    return 0;
}

I used the following code, based in the answer posted by @abc.

In addition, I was using Visual Studio Code and g++ to compile:

g ++ -std=c++0x main.cpp -lX11 'pkg-config --cflags opencv' 'pkg-config --libs opencv' -o main

The path for Xlib.h and Xutil.h (ubuntu 14.04): /usr/include/X11

Answer 4

I made a faster functor based screenshot code using Brandon's answer:

#include <opencv2/opencv.hpp>
#include <X11/Xlib.h>
#include <X11/Xutil.h>

class ScreenShot
{
    Display* display;
    Window root;
    int x,y,width,height;
    XImage* img{nullptr};
public:
    ScreenShot(int x, int y, int width, int height):
        x(x),
        y(y),
        width(width),
        height(height)
    {
        display = XOpenDisplay(nullptr);
        root = DefaultRootWindow(display);
    }

    void operator() (cv::Mat& cvImg)
    {
        if(img != nullptr)
            XDestroyImage(img);
        img = XGetImage(display, root, x, y, width, height, AllPlanes, ZPixmap);
        cvImg = cv::Mat(height, width, CV_8UC4, img->data);
    }

    ~ScreenShot()
    {
        if(img != nullptr)
            XDestroyImage(img);
        XCloseDisplay(display);
    }
};

And here is how you would use it in a loop (exit by pressing q):

int main()
{
    ScreenShot screen(0,0,1920,1080);
    cv::Mat img;

    while(true) 
    {
        screen(img);

        cv::imshow("img", img);
        char k = cv::waitKey(1);
        if (k == 'q')
            break;
    }
}

This is about 39% faster on my machine.