Perspective projection and view matrix: Both depth buffer and triangle face orientation are reversed in OpenGL

Question

I am having trouble with my scene in OpenGL. Objects that are supposed to be further away are drawn closer etc AND front facing triangles are being culled instead of back fa

Answer 1

There are some issues with in the calculation of the projection matrix. You have to adapt your code like this:

AV4X4FLOAT formProjMatrix(float FOVangle,float aspect,float nearz,float farz)
{
    AV4X4FLOAT A;

    A.m[0]  = 1.0 / (aspect*tanf(FOVangle/2));
    A.m[5]  = 1.0 / tanf(FOVangle/2);
    A.m[10] =  (nearz+farz)/(farz-nearz);
    A.m[11] = - 2.0 * nearz*farz/(farz-nearz);
    A.m[14] = - 1.0;
    return A;
}

The Perspective Projection Matrix looks like this:

r = right, l = left, b = bottom, t = top, n = near, f = far

2*n/(r-l)      0              0               0
0              2*n/(t-b)      0               0
(r+l)/(r-l)    (t+b)/(t-b)    -(f+n)/(f-n)   -1    
0              0              -2*f*n/(f-n)    0

it follows:

aspect = w / h
tanFov = tan( fov_y * 0.5 );

p[0][0] = 2*n/(r-l) = 1.0 / (tanFov * aspect)
p[1][1] = 2*n/(t-b) = 1.0 / tanFov

The following function will calculate the same projection matrix as gluPerspective or glm::perspective does:

#include <array>

const float cPI = 3.14159265f;
float ToRad( float deg ) { return deg * cPI / 180.0f; }

using TVec4  = std::array< float, 4 >;
using TMat44 = std::array< TVec4, 4 >;

TMat44 Perspective( float fov_y, float aspect )
{
    float fn = far + near
    float f_n = far - near;
    float r = aspect;
    float t = 1.0f / tan( ToRad( fov_y ) / 2.0f );

    return TMat44{ 
        TVec4{ t / r, 0.0f,  0.0f,                 0.0f },
        TVec4{ 0.0f,  t,     0.0f,                 0.0f },
        TVec4{ 0.0f,  0.0f, -fn / f_n,            -1.0f },
        TVec4{ 0.0f,  0.0f, -2.0f*far*near / f_n,  0.0f }
    };
}

On the viewport the X-axis points to the left, the Y-axis up and the Z-axis out of the view (Note in a right hand system the Z-Axis is the cross product of the X-Axis and the Y-Axis).

The following code does the same as gluLookAt or glm::lookAt does:

using TVec3  = std::array< float, 3 >;
using TVec4  = std::array< float, 4 >;
using TMat44 = std::array< TVec4, 4 >;

TVec3 Cross( TVec3 a, TVec3 b ) { return { a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0] }; }
float Dot( TVec3 a, TVec3 b ) { return a[0]*b[0] + a[1]*b[1] + a[2]*b[2]; }
void Normalize( TVec3 & v )
{
    float len = sqrt( v[0] * v[0] + v[1] * v[1] + v[2] * v[2] );
    v[0] /= len; v[1] /= len; v[2] /= len;
}

TMat44 Camera::LookAt( const TVec3 &pos, const TVec3 &target, const TVec3 &up )
{ 
    TVec3 mz = { pos[0] - target[0], pos[1] - target[1], pos[2] - target[2] };
    Normalize( mz );
    TVec3 my = { up[0], up[1], up[2] };
    TVec3 mx = Cross( my, mz );
    Normalize( mx );
    my = Cross( mz, mx );

    TMat44 v{
        TVec4{ mx[0], my[0], mz[0], 0.0f },
        TVec4{ mx[1], my[1], mz[1], 0.0f },
        TVec4{ mx[2], my[2], mz[2], 0.0f },
        TVec4{ Dot(mx, pos), Dot(my, pos), -Dot(mz, pos), 1.0f }
    };

    return v;
}

Adapt your code like this:

AV4X4FLOAT formViewModelMatrix(AV4FLOAT pos,AV4FLOAT target,AV4FLOAT up)
{ 
    AV4FLOAT mz;
    mz.x = pos.x - target.x; mz.y = pos.y - target.y; mz.z = pos.z - target.z; mz.w = 1.0f;
    mz.normalize();

    AV4FLOAT my;
    my.x = up.x; my.y = up.y; my.z = up.z; my.w = 1.0f;

    AV4FLOAT mx;
    mx.x = my.y*mz.z - my.z*mz.y; mx.y = my.z*mz.x - my.x*mz.z; mx.z = my.x*mz.y - my.y*mz.x; mx.w = 1.0f;
    mx.normylize();

    my.x = mz.y*mx.z - mz.z*mx.y; my.y = mz.z*mx.x - mz.x*mx.z; my.z = mz.x*mx.y - mz.y*mx.x; my.w = 1.0f;

    AV4FLOAT t;
    t.x = mx.x*pos.x + mx.y*pos.y + mx.z*pos.z; 
    t.y = my.x*pos.x + my.y*pos.y + my.z*pos.z; 
    t.z = -(mz.x*pos.x + mz.y*pos.y + mz.z*pos.z); 

    AV4X4FLOAT m;
    m[0]  = mx.x;  m[1]  = my.x;  m[2]  = mz.x;  m[3]  = 0.0f;
    m[4]  = mx.y;  m[5]  = my.y;  m[6]  = mz.y;  m[7]  = 0.0f;
    m[8]  = mx.z;  m[9]  = my.z;  m[10] = mz.z;  m[11] = 0.0f;
    m[12] = t.x;   m[13] = t.y;   m[14] = t.z;   m[15] = 1.0f;

    return m
}

See further the answers to the following question:

• How to render depth linearly in modern OpenGL with gl_FragCoord.z in fragment shader?
• How to recover view space position given view space depth value and ndc xy
• Transform the modelMatrix
• Stretching Issue with Custom View Matrix