Moderngl: project photo with angle

Question

I am trying to project a photo with some angle.

If the photo was taken when the camera was looking straight ahead, then the camera angles (yaw, pitch, roll) are all zero.

Now let's say that the camera was looking somewhat upwards, let's say with pitch=1 radians, then the photo is actually capturing a trapezoid and not a rectangle:

Now let's get to the code - this is a simple program that projects a photo with no angles, using moderngl-window:

import moderngl
import moderngl_window
import numpy as np
from PIL import Image


class Pygame(moderngl_window.WindowConfig):
    window_size = 1280, 720

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        self.program = self.ctx.program(
            vertex_shader="""
                #version 330
                in vec2 vertex_xy;
                in vec2 vertex_uv;
                uniform mat4 model;
                out vec2 fragment_uv; 
                void main() {
                    vec4 p = vec4(vertex_xy, 0.0, 1.0);
                    gl_Position = model * p;
                    fragment_uv = vertex_uv;
                }
                """,
            fragment_shader="""
                #version 330
                in vec2 fragment_uv;
                uniform sampler2D texture0;
                out vec4 fragment_color;
                void main() {
                    fragment_color = texture(texture0, fragment_uv);
                }
                """
        )
        self.program['model'].write(bytes(np.eye(4, dtype=np.float32)))
        self.program['texture0'].value = 0

        self.vertex_array = self.init_vertex_array(self.ctx, self.program)

        image = Image.open('test.jpg').transpose(Image.FLIP_TOP_BOTTOM)
        self.texture = self.ctx.texture(image.size, 3, image.tobytes())
        self.texture.use()

    def render(self, time, frametime):
        self.ctx.clear()
        self.vertex_array.render()

    def init_vertex_array(self, context: moderngl.Context, program: moderngl.Program) -> moderngl.VertexArray:
        vertices_xy = self.get_vertices_for_quad_2d(size=(2.0, 2.0), bottom_left_corner=(-1.0, -1.0))
        vertex_buffer_xy = context.buffer(vertices_xy.tobytes())

        vertices_uv = self.get_vertices_for_quad_2d(size=(1.0, 1.0), bottom_left_corner=(0.0, 0.0))
        vertex_buffer_uv = context.buffer(vertices_uv.tobytes())

        vertex_array = context.vertex_array(program, [(vertex_buffer_xy, "2f", "vertex_xy"),
                                                      (vertex_buffer_uv, "2f", "vertex_uv")])
        return vertex_array

    def get_vertices_for_quad_2d(self, size=(2.0, 2.0), bottom_left_corner=(-1.0, -1.0)) -> np.array:
        # A quad is composed of 2 triangles: https://en.wikipedia.org/wiki/Polygon_mesh
        w, h = size
        x_bl, y_bl = bottom_left_corner
        vertices = np.array([x_bl,     y_bl + h,
                             x_bl,     y_bl,
                             x_bl + w, y_bl,

                             x_bl,     y_bl + h,
                             x_bl + w, y_bl,
                             x_bl + w, y_bl + h], dtype=np.float32)
        return vertices


if __name__ == '__main__':
    moderngl_window.run_window_config(Pygame, args=('--window', 'glfw'))

When you run this program you will see this window:

Now, if we edit the render function to add an angle:

def render(self, time, frametime):
    pitch_rad = 1
    rotate_around_y_pitch = np.array([[np.cos(pitch_rad), 0, np.sin(pitch_rad), 0],
                                      [0, 1, 0, 0],
                                      [-np.sin(pitch_rad), 0, np.cos(pitch_rad), 0],
                                      [0, 0, 0, 1]], dtype=np.float32)
    self.program['model'].write(bytes(rotate_around_y_pitch))

    self.ctx.clear()
    self.vertex_array.render()

Then the projected photo will still be rectangular (just with a change in aspect ratio) and not a trapezoid.

What am I missing?

Answer 1

Many thanks to @Grimmy who supplied the missing details:

1. I should be using a projection matrix

2. I should position the object far from the camera

The complete working code:

import moderngl
import moderngl_window
import numpy as np
from PIL import Image

from pyrr import Matrix44


class Pygame(moderngl_window.WindowConfig):
    window_size = 1280, 720

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        self.program = self.ctx.program(
            vertex_shader="""
                #version 330
                in vec2 vertex_xy;
                in vec2 vertex_uv;
                uniform mat4 model;
                uniform mat4 projection;
                out vec2 fragment_uv; 
                void main() {
                    vec4 p = vec4(vertex_xy, 0.0, 1.0);
                    gl_Position = projection * model * p;
                    fragment_uv = vertex_uv;
                }
                """,
            fragment_shader="""
                #version 330
                in vec2 fragment_uv;
                uniform sampler2D texture0;
                out vec4 fragment_color;
                void main() {
                    fragment_color = texture(texture0, fragment_uv);
                }
                """
        )
        self.program['model'].write(bytes(np.eye(4, dtype=np.float32)))
        self.program['texture0'].value = 0

        self.vertex_array = self.init_vertex_array(self.ctx, self.program)

        image = Image.open('test.jpg').transpose(Image.FLIP_TOP_BOTTOM)
        self.texture = self.ctx.texture(image.size, 3, image.tobytes())
        self.texture.use()

    def render(self, time, frametime):
        pitch_rad = -1
        # Important! the -3 here positions the object far from the camera
        rotate_around_x_pitch = np.array([[1, 0, 0, 0],
                                          [0, np.cos(pitch_rad), -np.sin(pitch_rad), 0],
                                          [0, np.sin(pitch_rad), np.cos(pitch_rad), 0],
                                          [0, 0, -3, 1]], dtype=np.float32)

        projection = Matrix44.perspective_projection(45.0, self.aspect_ratio, 0.1, 1000.0, dtype="f4")
        self.program["projection"].write(projection)

        self.program['model'].write(bytes(rotate_around_x_pitch))

        self.ctx.clear()
        self.vertex_array.render()

    def init_vertex_array(self, context: moderngl.Context, program: moderngl.Program) -> moderngl.VertexArray:
        vertices_xy = self.get_vertices_for_quad_2d(size=(2.0, 2.0), bottom_left_corner=(-1.0, -1.0))
        vertex_buffer_xy = context.buffer(vertices_xy.tobytes())

        vertices_uv = self.get_vertices_for_quad_2d(size=(1.0, 1.0), bottom_left_corner=(0.0, 0.0))
        vertex_buffer_uv = context.buffer(vertices_uv.tobytes())

        vertex_array = context.vertex_array(program, [(vertex_buffer_xy, "2f", "vertex_xy"),
                                                      (vertex_buffer_uv, "2f", "vertex_uv")])
        return vertex_array

    def get_vertices_for_quad_2d(self, size=(2.0, 2.0), bottom_left_corner=(-1.0, -1.0)) -> np.array:
        # A quad is composed of 2 triangles: https://en.wikipedia.org/wiki/Polygon_mesh
        w, h = size
        x_bl, y_bl = bottom_left_corner
        vertices = np.array([x_bl,     y_bl + h,
                             x_bl,     y_bl,
                             x_bl + w, y_bl,

                             x_bl,     y_bl + h,
                             x_bl + w, y_bl,
                             x_bl + w, y_bl + h], dtype=np.float32)
        return vertices


if __name__ == '__main__':
    moderngl_window.run_window_config(Pygame, args=('--window', 'glfw'))