Audio appended to MediaSource buffer doesn't play immediately 100% of the time

Question

I am sending the same bit of audio every 5 seconds to my browser websocket client and appending it to the MediaSource buffer. The problem is sometimes the single piece of audio plays instantly (the desired functionality), other times it'll buffer several pieces and then play them all at once.

I can't find any reason why it decides one way or another, any thoughts?

    while True:
        await client.send_bytes(base64.b64decode(data()))
        await asyncio.sleep(5)

<audio></audio>
<script defer>
const simpleBuffer = []
const codec = 'audio/mpeg'
let audio = document.querySelector('audio')
ws = new WebSocket('ws://localhost:8000/ws/test');
ws.binaryType = "arraybuffer"
let mediaSource = new MediaSource()
audio.src = window.URL.createObjectURL(mediaSource)
audio.play()
mediaSource.onsourceopen = () => {
  sourceBuffer = mediaSource.addSourceBuffer(codec)
  console.log(sourceBuffer)
}

ws.onmessage = (d) => {
    simpleBuffer.push(d.data)
    appendToSourceBuffer()
    console.log(d)
}

const appendToSourceBuffer = () =>
{
  if (
    mediaSource.readyState === "open" &&
    sourceBuffer &&
    sourceBuffer.updating === false
  )
  {
    if(!simpleBuffer.length) {
      return
    }
    let data = simpleBuffer.shift()
    console.log(data)
    sourceBuffer.appendBuffer(data)
  } else {
    alert('setting timeout')
    setTimeout(appendToSourceBuffer, 500)
  }
}
</script>