OpenGL - draw pixels to screen?

Question

I want to draw a 2D array of pixel data (RGB / grayscale values) on the screen as fast as possible, using OpenGL. The pixel data changes frequently.

I had hoped that

Answer 1

My solution for getting dynamically changing image data to the screen in OpenGL,

#define WIN32_LEAN_AND_MEAN

#include "wx/wx.h"
#include "wx/sizer.h"
#include "wx/glcanvas.h"
#include "BasicGLPane.h"

// include OpenGL
#ifdef __WXMAC__
#include "OpenGL/glu.h"
#include "OpenGL/gl.h"
#else
#include <GL/glu.h>
#include <GL/gl.h>
#endif
#include "ORIScanMainFrame.h"

BEGIN_EVENT_TABLE(BasicGLPane, wxGLCanvas)
EVT_MOTION(BasicGLPane::mouseMoved)
EVT_LEFT_DOWN(BasicGLPane::mouseDown)
EVT_LEFT_UP(BasicGLPane::mouseReleased)
EVT_RIGHT_DOWN(BasicGLPane::rightClick)
EVT_LEAVE_WINDOW(BasicGLPane::mouseLeftWindow)
EVT_SIZE(BasicGLPane::resized)
EVT_KEY_DOWN(BasicGLPane::keyPressed)
EVT_KEY_UP(BasicGLPane::keyReleased)
EVT_MOUSEWHEEL(BasicGLPane::mouseWheelMoved)
EVT_PAINT(BasicGLPane::render)
END_EVENT_TABLE()

// Test data for image generation.  floats range 0.0 to 1.0, in RGBRGBRGB... order.
// Array is 1024 * 3 long. Note that 32 * 32 is 1024 and is the largest image we can randomly generate.
float* randomFloatRGB;
float* randomFloatRGBGrey;

BasicGLPane::BasicGLPane(wxFrame* parent, int* args) :
    wxGLCanvas(parent, wxID_ANY, args, wxDefaultPosition, wxDefaultSize, wxFULL_REPAINT_ON_RESIZE)
{
    m_context = new wxGLContext(this);

    randomFloatRGB = new float[1024 * 3];
    randomFloatRGBGrey = new float[1024 * 3];
    // In GL images 0,0 is in the lower left corner so the draw routine does a vertical flip to get 'regular' images right side up.
    for (int i = 0; i < 1024; i++) {
        // Red
        randomFloatRGB[i * 3] = static_cast <float> (rand()) / static_cast <float> (RAND_MAX);
        // Green
        randomFloatRGB[i * 3 + 1] = static_cast <float> (rand()) / static_cast <float> (RAND_MAX);
        // Blue
        randomFloatRGB[i * 3 + 2] = static_cast <float> (rand()) / static_cast <float> (RAND_MAX);
        // Telltale 2 white pixels in 0,0 corner.
        if (i < 2) {
            randomFloatRGB[i * 3] = randomFloatRGB[i * 3 + 1] = randomFloatRGB[i * 3 + 2] = 1.0f;
        }

        randomFloatRGBGrey[i * 3] = randomFloatRGB[i * 3];
        randomFloatRGBGrey[i * 3 + 1] = randomFloatRGB[i * 3];
        randomFloatRGBGrey[i * 3 + 2] = randomFloatRGB[i * 3];
    }

    // To avoid flashing on MSW
    SetBackgroundStyle(wxBG_STYLE_CUSTOM);
}

BasicGLPane::~BasicGLPane()
{
    delete m_context;
}

void BasicGLPane::resized(wxSizeEvent& evt)
{
    //  wxGLCanvas::OnSize(evt);
    Refresh();
}

int BasicGLPane::getWidth()
{
    return GetSize().x;
}

int BasicGLPane::getHeight()
{
    return GetSize().y;
}

void BasicGLPane::render(wxPaintEvent& evt)
{
    assert(GetParent());
    assert(GetParent()->GetParent());
    ORIScanMainFrame* mf = dynamic_cast<ORIScanMainFrame*>(GetParent()->GetParent());
    assert(mf);

    switch (mf->currentMainView) {
    case ORIViewSelection::ViewCamera:
        renderCamera(evt);
        break;
    case ORIViewSelection::ViewDepth:
        renderDepth(evt);
        break;
    case ORIViewSelection::ViewPointCloud:
        renderPointCloud(evt);
        break;
    case ORIViewSelection::View3DModel:
        render3DModel(evt);
        break;
    default:
        renderNone(evt);
    }
}

void BasicGLPane::renderNone(wxPaintEvent& evt) {
    if (!IsShown())
        return;
    SetCurrent(*(m_context));
    glPushAttrib(GL_ALL_ATTRIB_BITS);
    glClearColor(0.08f, 0.11f, 0.15f, 1.0f);
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
    glFlush();
    SwapBuffers();
    glPopAttrib();
}

GLuint makeOpenGlTextureFromDataLuninanceFloats(int width, int height, float* f) {
    GLuint textureID;

    glEnable(GL_TEXTURE_2D);
    glGenTextures(1, &textureID);

    // "Bind" the newly created texture : all future texture functions will modify this texture
    glBindTexture(GL_TEXTURE_2D, textureID);

    // Give the image to OpenGL
    glTexImage2D(GL_TEXTURE_2D, 0, GL_FLOAT, width, height, 0, GL_FLOAT, GL_LUMINANCE, f);

    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);

    return textureID;
}

GLuint makeOpenGlTextureFromRGBInts(int width, int height, unsigned int* f) {
    GLuint textureID;


    glEnable(GL_TEXTURE_2D);
    glGenTextures(1, &textureID);

    // "Bind" the newly created texture : all future texture functions will modify this texture
    glBindTexture(GL_TEXTURE_2D, textureID);

    // Give the image to OpenGL
    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, width, height, 0, GL_RGBA, GL_UNSIGNED_INT, f);

    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);

    return textureID;
}

/// <summary>
/// Range of each float is 0.0f to 1.0f
/// </summary>
/// <param name="width"></param>
/// <param name="height"></param>
/// <param name="floatRGB"></param>
/// <returns></returns>
GLuint makeOpenGlTextureFromRGBFloats(int width, int height, float* floatRGB) {
    GLuint textureID;

    // 4.6.0 NVIDIA 457.30  (R Keene machine, 11/25/2020)
    // auto sss = glGetString(GL_VERSION);

    glGenTextures(1, &textureID);

    // "Bind" the newly created texture : all future texture functions will modify this texture
    glBindTexture(GL_TEXTURE_2D, textureID);

    // Give the image to OpenGL
    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, width, height, 0, GL_RGB, GL_FLOAT, floatRGB);

    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);

    return textureID;
}

void BasicGLPane::DrawTextureToScreenFloat(int w, int h, float* floatDataPtr, GLuint (*textureFactory)(int width, int height, float* floatRGB)) {
    if (w <= 0 || h <= 0 || floatDataPtr == NULL || w > 5000 || h > 5000) {
        assert(false);
        return;
    }

    SetCurrent(*(m_context));

    glPushAttrib(GL_ALL_ATTRIB_BITS);
    glPushMatrix();
    glPushClientAttrib(GL_CLIENT_ALL_ATTRIB_BITS);

    glClearColor(0.15f, 0.11f, 0.02f, 1.0f);
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

    glEnable(GL_TEXTURE_2D);

    glLoadIdentity();
    glMatrixMode(GL_MODELVIEW);
    glLoadIdentity();
    glMatrixMode(GL_PROJECTION);
    glLoadIdentity();

    // 4.6.0 NVIDIA 457.30  (R Keene machine, 11/25/2020)
    // auto sss = glGetString(GL_VERSION);

    float onePixelW = (float)getWidth() / (float)w;
    float onePixelH = (float)getHeight() / (float)h;
    float orthoW = w;
    float orthoH = h;
    if (onePixelH > onePixelW) {
        orthoH = h * onePixelH / onePixelW;
    }
    else {
        orthoW = w * onePixelW / onePixelH;
    }
    // We want the image at the top of the window, not the bottom if the window is too tall.
    int topOfScreen = (float)getHeight() / onePixelH;

    // If the winjdow resizes after creation you need to change the viewport.
    glViewport(0, 0, getWidth(), getHeight());
    gluOrtho2D(0.0, orthoW, (double)topOfScreen - (double)orthoH, topOfScreen);

    GLuint myTextureName = textureFactory(w, h, floatDataPtr);

    glBegin(GL_QUADS);
    {
        // This order of UV coords and verticies will do the vertical flip of the image to get the 'regular' image 0,0
        // in the top left corner.
        glTexCoord2f(0.0f, 1.0f); glVertex3f(0.0f, 0.0f, 0.0f);
        glTexCoord2f(1.0f, 1.0f); glVertex3f(0.0f + w, 0.0f, 0.0f);
        glTexCoord2f(1.0f, 0.0f); glVertex3f(0.0f + w, 0.0f + h, 0.0f);
        glTexCoord2f(0.0f, 0.0f); glVertex3f(0.0f, 0.0f + h, 0.0f);
    }
    glEnd();

    glDeleteTextures(1, &myTextureName);

    glFlush();
    SwapBuffers();


    glPopClientAttrib();
    glPopMatrix();
    glPopAttrib();
}

void BasicGLPane::DrawTextureToScreenMat(wxPaintEvent& evt, cv::Mat m, float brightness) {
    m.type();
    if (m.empty()) {
        renderNone(evt);
        return;
    }

    if (m.type() == CV_32FC1) { // Grey scale.
        DrawTextureToScreenFloat(m.cols, m.rows, (float*)m.data, makeOpenGlTextureFromDataLuninanceFloats);
    }
    if (m.type() == CV_32FC3) { // Color.
        DrawTextureToScreenFloat(m.cols, m.rows, (float*)m.data, makeOpenGlTextureFromRGBFloats);
    }
    else {
        renderNone(evt);
    }
}

void BasicGLPane::renderCamera(wxPaintEvent& evt) {
    if (!IsShown())
        return;
    DrawTextureToScreenMat(evt, ORITopControl::Instance->im_white);
}

void BasicGLPane::renderDepth(wxPaintEvent& evt) {
    if (!IsShown())
        return;
    DrawTextureToScreenMat(evt, ORITopControl::Instance->depth_map);
}

void BasicGLPane::render3DModel(wxPaintEvent& evt) {
    if (!IsShown())
        return;
    SetCurrent(*(m_context));
    glPushAttrib(GL_ALL_ATTRIB_BITS);
    glPushMatrix();

    glClearColor(0.08f, 0.11f, 0.15f, 1.0f);
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);



    glFlush();
    SwapBuffers();

    glPopMatrix();
    glPopAttrib();
}

void BasicGLPane::renderPointCloud(wxPaintEvent& evt) {
    if (!IsShown())
        return;
    boost::unique_lock<boost::mutex> lk(ORITopControl::Instance->pointCloudCacheMutex);

    SetCurrent(*(m_context));
    glPushAttrib(GL_ALL_ATTRIB_BITS);
    glPushMatrix();

    glLoadIdentity();
    glMatrixMode(GL_MODELVIEW);
    glLoadIdentity();
    glMatrixMode(GL_PROJECTION);
    glLoadIdentity();

    glViewport(0, 0, getWidth(), getHeight());

    glClearColor(0.08f, 0.11f, 0.15f, 1.0f);
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

    if (ORITopControl::Instance->pointCloudCache.size() > 0) {
        glMatrixMode(GL_PROJECTION);
        gluPerspective( /* field of view in degree */ 40.0,
            /* aspect ratio */ 1.0,
            /* Z near */ 1.0, /* Z far */ 500.0);
        glMatrixMode(GL_MODELVIEW);
        gluLookAt(100, 70, 200, // Eye
            25, 25, 25, // Look at pt
            0, 0, 1); // Up Vector

        glPointSize(2.0);
        glBegin(GL_POINTS);
        // Use explicit for loop because pointCloudFragments can grow asynchronously.
        for (int i = 0; i < ORITopControl::Instance->pointCloudCache.size(); i++) {
            auto frag = ORITopControl::Instance->pointCloudCache[i];
            auto current_point_cloud_ptr = frag->cloud;
            glPushMatrix();
            // glMultMatrixf(frag->xform.data());
            for (size_t n = 0; n < current_point_cloud_ptr->size(); n++) {
                glColor3ub(255, 255, 255);
                glVertex3d(current_point_cloud_ptr->points[n].x, current_point_cloud_ptr->points[n].y, current_point_cloud_ptr->points[n].z);
            }
            glPopMatrix();
        }
        glEnd();
    }

    glFlush();
    SwapBuffers();

    glPopMatrix();
    glPopAttrib();
}

Answer 2

I would think the fastest way would be to draw a screen sized quad with ortho projection and use a pixel shader and Texture Buffer Object to draw directly to the texture in the pixel shader. Due to latency transferring to/from the TBO you may want to see if double buffering would help.

If speed isn't much of a concern (you just need fairly interactive framerates) glDrawPixels is easy to use and works well enough for many purposes.

Answer 3

Maybe glDrawPixels is the function you are looking for? Though if the data is static it would be better to create a texture with it, and then draw that each frame.

Answer 4

I recently had a similar problem, as I am trying to render a video to screen (ie repeatedly upload pixel data to the VRAM), my approach is:

• use glTexImage2D and glTexSubImage2D to upload the data to the texture (ie bind the texture (and texture unit, if applicable) before calling that)

• in my case as the video frame rate (usually about 24 fps) is lower than the framerate of my application (aimed at 60 fps), in order to avoid uploading the same data again I use a framebuffer object (check out glGenFramebuffers/glBindFramebuffer/glDeleteFramebuffers) and link my texture with the framebuffer (glFramebufferTexture2D). I then upload that texture once, and draw the same frame multiple times (just normal texture access with glBindTexture)

• I don't know which platform you are using, but as I am targetting Mac I use some Apple extensions to ensure the data transfer to the VRAM happens through DMA (ie make glTexSubImage2D return immediately to let the CPU do other work) - please feel free to ask me for more info if you are using Mac too

• also as you are using just grayscale, you might want to consider just using a GL_LUMINANCE texture (ie 1 byte per pixel) rather than RGB based format to make the upload faster (but that depends on the size of your texture data, I was streaming HD 1920x1080 video so I needed to make sure to keep it down)

• also be aware of the format your hardware is using to avoid unnecessary data conversions (ie normally it seems better to use BGRA data than for example just RGB)

• finally, in my code I replaced all the fixed pipeline functionality with shaders (in particular the conversion of the data from grayscale or YUV format to RGB), but again all that depends on the size of your data, and the workload of your CPU or GPU

Hope this helps, feel free to message me if you need further info