Mouse picking miss

Question

I did mouse picking with terrain for these lessons (but used c++)

https://www.youtube.com/watch?v=DLKN0jExRIM&index=29&listhLoLuZVfUksDP http://antongerdelan.net/opengl/raycasting.html

The problem is that the position of the mouse does not correspond to the place where the ray intersects with the terrane: There's a big blunder on the vertical and a little horizontal. Do not look at the shadows, this is not a corrected map of normals. What can be wrong? My code:

  void MousePicker::update() {

  view = cam->getViewMatrix();

  currentRay = calculateMouseRay();
  if (intersectionInRange(0, RAY_RANGE, currentRay)) {
    currentTerrainPoint = binarySearch(0, 0, RAY_RANGE, currentRay);
  }
  else {
    currentTerrainPoint = vec3();
  }
}
vec3 MousePicker::calculateMouseRay() {    
  glfwGetCursorPos(win, &mouseInfo.xPos, &mouseInfo.yPos);
  vec2 normalizedCoords = getNormalizedCoords(mouseInfo.xPos, mouseInfo.yPos);
  vec4 clipCoords = vec4(normalizedCoords.x, normalizedCoords.y, -1.0f, 1.0f);
  vec4 eyeCoords = toEyeCoords(clipCoords);
  vec3 worldRay = toWorldCoords(eyeCoords);

  return worldRay;
}

vec2 MousePicker::getNormalizedCoords(double xPos, double yPos) {
  GLint width, height;
  glfwGetWindowSize(win, &width, &height);
  //GLfloat x = (2.0 * xPos) / width  - 1.0f;   
  GLfloat x = -((width - xPos) / width - 0.5f) * 2.0f;
  //GLfloat y = 1.0f - (2.0f * yPos) / height;
  GLfloat y = ((height - yPos) / height - 0.5f) * 2.0f;
  //float z = 1.0f;
  mouseInfo.normalizedCoords = vec2(x, y);

  return vec2(x,y);
}

vec4 MousePicker::toEyeCoords(vec4 clipCoords) {
  vec4 invertedProjection = inverse(projection) * clipCoords;
  //vec4 eyeCoords = translate(invertedProjection, clipCoords);
  mouseInfo.eyeCoords = vec4(invertedProjection.x, invertedProjection.y, -1.0f, 0.0f);
  return vec4(invertedProjection.x, invertedProjection.y, -1.0f, 0.0f);
}

vec3 MousePicker::toWorldCoords(vec4 eyeCoords) {
  vec3 rayWorld = vec3(inverse(view) * eyeCoords);
  vec3 mouseRay = vec3(rayWorld.x, rayWorld.y, rayWorld.z);
  rayWorld = normalize(rayWorld);
  mouseInfo.worldRay = rayWorld;
  return rayWorld;
}

//*********************************************************************************

vec3 MousePicker::getPointOnRay(vec3 ray, float distance) {
  vec3 camPos = cam->getCameraPos();  
  vec3 start = vec3(camPos.x, camPos.y, camPos.z);
  vec3 scaledRay = vec3(ray.x * distance, ray.y * distance, ray.z * distance);
  return vec3(start + scaledRay);
}

vec3 MousePicker::binarySearch(int count, float start, float finish, vec3 ray) {
  float half = start + ((finish - start) / 2.0f);
  if (count >= RECURSION_COUNT) {
    vec3 endPoint = getPointOnRay(ray, half);
    //Terrain* ter = &getTerrain(endPoint.x, endPoint.z);
    if (terrain != NULL) {
      return endPoint;
    }
    else {
      return vec3();
    }
  }

  if (intersectionInRange(start, half, ray)) {
    return binarySearch(count + 1, start, half, ray);
  }
  else {
    return binarySearch(count + 1, half, finish, ray);
  }  
}

bool MousePicker::intersectionInRange(float start, float finish, vec3 ray) {
  vec3 startPoint = getPointOnRay(ray, start);
  vec3 endPoint = getPointOnRay(ray, finish);
  if (!isUnderGround(startPoint) && isUnderGround(endPoint)) {
    return true;
  }
  else {
    return false;
  }  
}

bool MousePicker::isUnderGround(vec3 testPoint) {
  //Terrain* ter = &getTerrain(testPoint.x, testPoint.z);
  float height = 0;
  if (terrain != NULL) {
    height = terrain->getHeightPoint(testPoint.x, testPoint.z);
    mouseInfo.height = height;
  }
  if (testPoint.y < height) {
    return true;
  }
  else {
    return false;
  }

}

Terrain MousePicker::getTerrain(float worldX, float worldZ) {
  return *terrain;
}

Answer 1

In perspective projection, a ray from the eye position through a point on the screen can defined by 2 points. The first point is the eye (camera) position which is (0, 0, 0) in view space. The second point has to be calculated by the position on the screen.
The screen position has to be converted to normalized device coordinates in range from (-1,-1) to (1,1).

w = with of the viewport
h = height of the viewport
x = X position of the mouse
y = Y position ot the mouse

GLfloat ndc_x = 2.0 * x/w - 1.0;
GLfloat ndc_y = 1.0 - 2.0 * y/h; // invert Y axis

To calculate a point on the ray, which goes through the camera position and through the point on the screen, the field of view and the aspect ratio of the perspective projection has to be known:

fov_y  = vertical field of view angle in radians
aspect = w / h

GLfloat   tanFov = tan( fov_y * 0.5 );
glm::vec3 ray_P  = vec3( ndc_x * aspect * tanFov, ndc_y * tanFov, -1.0 ) );

A ray from the camera position through a point on the screen can be defined by the following position (P0) and normalized direction (dir), in world space:

view = view matrix

glm::mat4 invView = glm::inverse( view );

glm::vec3 P0  = invView * glm::vec3(0.0f, 0.0f, 0.0f); 
           // = glm::vec3( view[3][0], view[3][1], view[3][2] ); 

glm::vec3 dir = glm::normalize( invView * ray_P - P0 );

In this case, the answers to the following questions will be interesting too:

• How to recover view space position given view space depth value and ndc xy
• Is it possble get which surface of cube will be click in OpenGL?
• How to render depth linearly in modern OpenGL with gl_FragCoord.z in fragment shader?
• GLSL spotlight projection volume

Applying to your code results in the following changes:

The Perspective Projection Matrix looks like this:

r = right, l = left, b = bottom, t = top, n = near, f = far

2*n/(r-l)      0              0               0
0              2*n/(t-b)      0               0
(r+l)/(r-l)    (t+b)/(t-b)    -(f+n)/(f-n)   -1    
0              0              -2*f*n/(f-n)    0

it follows:

aspect = w / h
tanFov = tan( fov_y * 0.5 );

p[0][0] = 2*n/(r-l) = 1.0 / (tanFov * aspect)
p[1][1] = 2*n/(t-b) = 1.0 / tanFov

Convert from screen (mouse) coordinates to normalized device coordinates:

vec2 MousePicker::getNormalizedCoords(double x, double y) {

    GLint w, h;
    glfwGetWindowSize(win, &width, &height);

    GLfloat ndc_x = 2.0 * x/w - 1.0;
    GLfloat ndc_y = 1.0 - 2.0 * y/h; // invert Y axis
    mouseInfo.normalizedCoords = vec2(ndc_x, ndc_x);
    return vec2(ndc_x, ndc_x);
}

Calculate A ray from the camera position through a point on the screen (mouse position) in world space:

vec3 MousePicker::calculateMouseRay( void ) {    
    glfwGetCursorPos(win, &mouseInfo.xPos, &mouseInfo.yPos);
    vec2 normalizedCoords = getNormalizedCoords(mouseInfo.xPos, mouseInfo.yPos);

    ray_Px = normalizedCoords.x / projection[0][0]; // projection[0][0] == 1.0 / (tanFov * aspect)
    ray_Py = normalizedCoords.y / projection[1][1]; // projection[1][1] == 1.0 / tanFov
    glm::vec3 ray_P = vec3( ray_Px, ray_Py, -1.0f ) );

    vec3      camPos  = cam->getCameraPos();  // == glm::vec3( view[3][0], view[3][1], view[3][2] );     
    glm::mat4 invView = glm::inverse( view ); 

    glm::vec3 P0  = camPos; 
    glm::vec3 dir = glm::normalize( invView * ray_P - P0 );
    return dir;
}