How to properly implement an audio latency test with pygame

Question

I have a school project that involves making a simple rhythm game in python. I am currently trying to synchronise a flashing dot to a song of a known bpm (120 in this case) using pygame.

The flashing dot is synchronised at first but slowly desyncs with time. I have searched on internet how could a rhythm game be made and I found that I needed to do an audio latency test. However, the code that i came up with is not really working and so I would really like some help with that. I already tried removing the gap at the beginning of my mp3 and exported it as an ogg with no metadata.

import sys, pygame, math, random, time

pygame.init()

size = width, height = 1000, 1000
speed = [2, 2]
black = 0, 0, 0

screen = pygame.display.set_mode(size)
pygame.display.update()
pygame.mixer.init()
pygame.mixer.music.load("metr.ogg")

continuer = True

bpm = 120

t = 60 / bpm

font = pygame.font.Font('freesansbold.ttf', 15)

lasttime = 0

delay = 0
red = (200, 0, 0)
green = (0, 200, 0)
colors = [red, red, red, red]
i = 0
av = []
drawn = False
clock = pygame.time.Clock()
pygame.mixer.music.play()
while continuer:

    time = (pygame.mixer.music.get_pos()) / 1000
    nextkey = lasttime + t + (delay / 1000)
    if time > nextkey:
        if drawn:
            screen.fill((0, 0, 0))
            drawn = False
        else:
            pygame.draw.circle(screen, colors[i], (500, 500), 20)
            drawn = True
            if i == 3:
                i = 0
            else:
                i += 1
        lasttime += t
    screen.blit(font.render('delay = ' + str(delay) + "ms", True, (255, 255, 255), (0, 0, 0)), (500, 600))
    for event in pygame.event.get():
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_RIGHT:
                delay += 1
            elif event.key == pygame.K_h:
                av.append(nextkey - time)
                delay = sum(av) / len(av)
            elif event.key == pygame.K_LEFT:
                delay -= 1
            elif event.key == pygame.K_SPACE:
                continuer = False
    pygame.display.flip()
    clock.tick(60)

pygame.quit()

The file metr.ogg is just a metronome at 120bpm

Answer 1

It's because your variable nextkey is a summation of different variables, each of which will contain a small error. So you're always adding on small errors until it eventually becomes noticeable.

The way to prevent accumulative errors is to always do calculations directly from the source. In your case pygame.mixer.music.get_pos().

In the example below, we first create a variable ms_per_beat. This value is a constant that defines how many milliseconds should pass for each beat. For each frame, we calculate time_since_last_beat based on how long the song has been played. A rough, incomplete example:

bpm = 120
beats_per_ms = (bpm / 60) / 1000
ms_per_beat  = 1 / beats_per_ms  # How many milliseconds each beat takes.
current_beat = 0                 # The beat we're currently on.
time_since_last_beat = 0         # How many milliseconds since last beat.

dot_display_time = 500  # Display the dot for 500 ms (half a second)
dot_timer = 0           # Keeps track on how long the dot has been displayed.
display_dot = False     # Whether to display the dot or not.

clock = pygame.time.Clock()

while running:
    dt = clock.tick(60)

    current_play_time_ms = pygame.mixer.music.get_pos() / 1000
    time_since_last_beat = current_play_time_ms - (current_beat * ms_per_beat)

    if time_since_last_beat >= ms_per_beat:
        print('Bop!')
        current_beat += 1
        display_dot = True
        dot_timer = dot_display_time

    screen.fill((0, 0, 0))

    if display_dot:
        dot_timer -= dt
        if dot_timer <= 0:
            display_dot = False
        pygame.draw.circle(screen, colors[i], (500, 500), 20)

    pygame.display.update()

The example above assumes that the audio starts exactly on the beat. It also assumes that it actually is 120 bpm. If it's 120.1 bpm, it'll get out of sync eventually. A more proper way would be to analyze the audio for a peak in amplitude and just display the dot then. If your audio is just a metronome, then it could be done with pygame.mixer.music.get_volume().

dot_display_time = 500  # Display the dot for 500 ms (half a second)
dot_timer = 0           # Keeps track on how long the dot has been displayed.
display_dot = False     # Whether to display the dot or not.

beat_volume_threshold = 0.7  # The volume the audio has to overcome to count as a beat.

clock = pygame.time.Clock()

while running:
    dt = clock.tick(60)

    if pygame.mixer.music.get_volume() >= beat_volume_threshold:
        print('Bop!')
        display_dot = True
        dot_timer = dot_display_time

    screen.fill((0, 0, 0))

    if display_dot:
        dot_timer -= dt
        if dot_timer <= 0:
            display_dot = False
        pygame.draw.circle(screen, colors[i], (500, 500), 20)

    pygame.display.update()

However, I would recommend a totally different approach. Synchronization is always a hassle, so try to avoid it whenever possible. Instead of synching your game with a metronome, let your game be the metronome. Have a single "bop" audio that you play whenever a certain time has passed. Then your audio and graphics will always be synchronized since both use the same clock. The example below plays the audio be scheduling a user event with the help of pygame.time.set_timer.

import pygame

# This fixes the latency issue with the pygame mixer.
pygame.mixer.pre_init(22050, -16, 2, 1024)
pygame.init()

PLAY_CLICK = pygame.USEREVENT + 1

screen = pygame.display.set_mode((400, 400))

dot_display_time = 200  # Display the dot for 500 ms (half a second)
dot_timer = 0           # Keeps track on how long the dot has been displayed.
display_dot = False     # Whether to display the dot or not.

bpm = 120
beats_per_ms = (bpm / 60) / 1000
ms_per_beat  = 1 / beats_per_ms  # How many milliseconds each beat takes.

clock = pygame.time.Clock()
sound = pygame.mixer.Sound('bop.wav')

pygame.time.set_timer(PLAY_CLICK, int(ms_per_beat))  # Play sound repeatedly every 'beats_per_ms'.

running = True
while running:
    dt = clock.tick(60)

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False
        elif event.type == PLAY_CLICK:
            sound.play()
            display_dot = True
            dot_timer = dot_display_time

    screen.fill((0, 0, 0))

    if display_dot:
        dot_timer -= dt
        if dot_timer <= 0:
            display_dot = False
        pygame.draw.circle(screen, pygame.Color('green'), (200, 200), 20)

    pygame.display.update()

If you not comfortable with scheduling events, you could calculate the timing yourself.

import pygame

# This fixes the latency issue with the pygame mixer.
pygame.mixer.pre_init(22050, -16, 2, 1024)
pygame.init()

font = pygame.font.Font('freesansbold.ttf', 15)
screen = pygame.display.set_mode((400, 400))

dot_display_time = 250  # Display the dot for 250 ms (quarter of a second)
dot_timer = 0           # Keeps track on how long the dot has been displayed.
display_dot = False     # Whether to display the dot or not.

bpm = 120

clock = pygame.time.Clock()
sound = pygame.mixer.Sound('bop.wav')

time = 0

running = True
while running:
    dt = clock.tick(60)
    time += dt

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False
        elif event.type == pygame.KEYUP:
            if event.key == pygame.K_LEFT:
                bpm -= 5
            elif event.key == pygame.K_RIGHT:
                bpm += 5

    # Calculate how long to wait based on the bpm.
    beats_per_ms = (bpm / 60) / 1000
    ms_per_beat  = 1 / beats_per_ms  # How many milliseconds each beat takes.

    if time >= ms_per_beat:
        time -= ms_per_beat
        sound.play()
        display_dot = True
        dot_timer = dot_display_time

    screen.fill((0, 0, 0))
    screen.blit(font.render('BPM {}'.format(bpm), True, (255, 255, 255), (0, 0, 0)), (160, 20))

    if display_dot:
        dot_timer -= dt
        if dot_timer <= 0:
            display_dot = False
        pygame.draw.circle(screen, pygame.Color('green'), (200, 200), 20)

    pygame.display.update()

Note however that pygame seems to have difficulties when it comes to playing sound at a certain timing. It works better if you decrease the buffer (as explained in the documentation).

pygame.mixer.pre_init(22050, -16, 2, 1024)
pygame.init()