Camera placement for Kinect using Nuitrack

134 Views Asked by At

I have screen space coordinates from Kinect. I need to get the world space coordinates from kinect for my opengl camera; or I need to place my camera in the same position as Kinect so that I can see the points that I get from Kinect. I think I will need the world coordinates w.r.t. my opengl camera for getting collision detection to other objects that I am drawing too. How do I get this ScreenspaceToWorld Mapping ?

I am using this answer as reference, but I dont have the position & orientation of Kinect to calculate the View and Projection matrix.

Edit :

Kinect FOV

I need to be able to render objects in same coordinate space as Kinect's. In order to do so, I am using projection matrix using these values :

Near Plane : 0.01f
Far Plane : 1700.0f
Horizontal FOV : 60.0f
Vertical FOV : 45.0f
Camera Position: (0.0f, 2.0f, 3.0f)
Camera Direction: (0.0f, 2.0f, -3.0f)

Using these values, I am not able to see any object rendered. Here is my rendering code :

GameRenderer::GameRenderer()
{
    tracker = new GestureTracker();
    tracker->init();
    tracker->startGestureDetection();
}

bool GameRenderer::Init(int argc, char* argv[])
{        
    GLfloat points[] = { 0.0f, 0.5f, 0.0f, 0.5f, -0.5f, 0.0f, -0.5f, -0.5f, 0.0f };
    GLfloat point[] = { 0.0f, 0.0f, 0.0f };
    glGenBuffers(1, &m_vbo);
    glBindBuffer(GL_ARRAY_BUFFER, m_vbo);

    glBufferData(GL_ARRAY_BUFFER, sizeof(points), points, GL_DYNAMIC_DRAW);
    // vao
    glGenVertexArrays(1, &m_vao);
    glBindVertexArray(m_vao);
    glEnableVertexAttribArray(0);
    glBindBuffer(GL_ARRAY_BUFFER, m_vbo);
    glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 0, NULL);


    const char *vertex_shader = "#version 410\n"
                                "in vec3 vp;"
                                "uniform mat4 model_matrix;"
                                "uniform mat4 view_matrix;"
                                "uniform mat4 projection_matrix;"
                                "void main () {"
                                "  gl_Position = projection_matrix * view_matrix * model_matrix * vec4(vp, 1.0);"
                                "}";

    const char *fragment_shader = "#version 410\n"
                                  "out vec4 frag_colour;"
                                  "void main () {"
                                  "  frag_colour = vec4(0.0f, 0.5f, 1.0f, 1.0f);"
                                  "}";

    m_vert_shader = glCreateShader(GL_VERTEX_SHADER);
    glShaderSource(m_vert_shader, 1, &vertex_shader, NULL);
    glCompileShader(m_vert_shader);
    int params = -1;
    glGetShaderiv( m_vert_shader, GL_COMPILE_STATUS, &params );
    if ( GL_TRUE != params ) {
        ShaderLog(m_vert_shader);
        return 1;
    }
    m_frag_shader = glCreateShader(GL_FRAGMENT_SHADER);
    glShaderSource(m_frag_shader, 1, &fragment_shader, NULL);
    glCompileShader(m_frag_shader);
    params = -1;
    glGetShaderiv( m_frag_shader, GL_COMPILE_STATUS, &params );
    if ( GL_TRUE != params ) {
        ShaderLog(m_frag_shader);
        return 1;
    }
    m_shader_program = glCreateProgram();
    glAttachShader(m_shader_program, m_frag_shader);
    glAttachShader(m_shader_program, m_vert_shader);
    glLinkProgram(m_shader_program);

    this->m_model_matrix = glm::mat4(1.0f);
    m_model_matrix = glm::scale(m_model_matrix, glm::vec3(4.0f));
    // m_model_matrix = glm::rotate(m_model_matrix, glm::radians(10.0f), glm::vec3(0.0f));
    m_model_matrix = glm::translate(m_model_matrix, glm::vec3(0.0f));
    this->m_view_matrix = glm::mat4(1.0f);
    this->m_proj_matrix = glm::mat4(1.0f);

    GLuint model_mat_location = glGetUniformLocation(this->m_shader_program, "model_matrix");
    glUseProgram(this->m_shader_program);
    glUniformMatrix4fv(model_mat_location, 1, GL_FALSE, glm::value_ptr(this->m_model_matrix));
    return 1;
}

void GameRenderer::Draw(Camera& camera)
{
    auto hands = tracker->getHands();

    if(hands.size()) {
        Point3f coordinates = hands[0].getHandPosition();
        glBindBuffer(GL_ARRAY_BUFFER, m_vbo);
        GLfloat hand_coordinate[3] = { coordinates.x, coordinates.y, coordinates.z };
        std::cout << coordinates.x << " " << coordinates.y << " " << coordinates.z << std::endl;
        // glBufferData(GL_ARRAY_BUFFER, sizeof(GLfloat) * 3, hand_coordinate, GL_DYNAMIC_DRAW);
        this->m_model_matrix = glm::mat4(1.0f);
        m_model_matrix = glm::scale(m_model_matrix, glm::vec3(10.0f));
        m_model_matrix = glm::translate(m_model_matrix, glm::vec3(coordinates.x, coordinates.y, coordinates.z));
        glUseProgram(this->m_shader_program);
        GLuint loc = glGetUniformLocation(this->m_shader_program, "model_matrix");
        glUniformMatrix4fv(loc, 1, GL_FALSE, glm::value_ptr(this->m_model_matrix));
        glUseProgram(0);
    }
    this->m_view_matrix = camera.GetViewMatrix();
    this->m_proj_matrix = glm::perspective(glm::radians(camera.GetFieldOfView()),
                                           60.0f/45.0f, camera.GetNearPlane(), camera.GetFarPlane());


    GLuint view_mat_location = glGetUniformLocation(this->m_shader_program, "view_matrix");
    glUseProgram(this->m_shader_program);
    glUniformMatrix4fv(view_mat_location, 1, GL_FALSE, glm::value_ptr(this->m_view_matrix));

    GLuint proj_mat_location = glGetUniformLocation(this->m_shader_program, "projection_matrix");
    glUseProgram(this->m_shader_program);
    glUniformMatrix4fv(proj_mat_location, 1, GL_FALSE, glm::value_ptr(this->m_proj_matrix));

    CheckGLError();
    glUseProgram(m_shader_program);
    glBindVertexArray(m_vao);
    glBindBuffer(GL_ARRAY_BUFFER, m_vbo);
    // glPointSize(50.0f);
    glDrawArrays(GL_TRIANGLES, 0, 3);
    glUseProgram(0);
    CheckGLError();
}

Coordinate values from Kinect's depth frame (these are hand coordinates as tracked by Kinect) are :

x y z
-75.1628 136.374 650.176
-73.7582 141.239 665.259
-69.9152 148.691 702.163
-67.1218 151.315 723.226
-64.4887 153.013 745.074
-62.3365 153.783 767.393
-62.3365 153.783 767.393
-60.1887 152.979 789.881
-57.9948 150.762 812.395
-55.8624 146.575 834.187
-56.2673 133.543 873.035

When Hands are detected by Kinect, my geometry (a triangle right now), stops getting rendered, although it gets rendered before (since it would be at origin).

What needs to be corrected in my view/projection matrix or code to be able to render in same space as Kinect's ?

0

There are 0 best solutions below