Why does ffmpeg parses an entire file beforehand

Question

When calling avformat_open_input with AVIOContext set it seems that the library parses an entire file. I set both read and seek functions on the context. The video file is a fragmented mp4 with a lot of moof boxes. The file is located on a remote server. Each seek call is transformed into an http request. The library issues one seek for each moof. Is there any way to prevent avformat_open_input from parsing the entire file beforehand? I would like to open the file and decode one frame at a specific time offset. Any suggestions?

Here is the snippet of GO code

func (d *videoDecoder) init() error {
    avio_ctx_buffer := (*Cuint8_t)(lib.av_mallocz(avio_buffer_size))

    d.format_ctx = lib.avformat_alloc_context()
    d.avio_ctx = lib.avio_alloc_context(avio_ctx_buffer, Cint(avio_buffer_size), 0, uintptr(unsafe.Pointer(&d.ReadSeeker)), (Crw_packet_callback)(C.read_packet), nil, (Cseek_packet_callback)(C.seek_packet))
    d.format_ctx.pb = d.avio_ctx

    if ret := lib.avformat_open_input(&d.format_ctx, nil, nil, &d.options); ret < 0 {
        return fmt.Errorf("avformat open input: %w", convertRetCode(ret))
    }

    return nil
}

func read_packet(opaque unsafe.Pointer, buffer *Cuint8_t, size Cint) Cint {
    r := *(*io.ReadSeeker)(opaque)

    gobuf := (*[1 << 30]byte)(unsafe.Pointer(buffer))[:size:size]
    fmt.Printf("REQUESTED TO READ: %v\n", size)

    n, err := r.Read(gobuf)
    if err != nil {
        if errors.Is(err, io.EOF) {
            return CAVERROR_EOF
        }
        fmt.Printf("error reading from reader: %v\n", err)
    }

    return Cint(n)
}

//export seek_packet
func seek_packet(opaque unsafe.Pointer, offset Cint64_t, whence Cint) Cint64_t {
    r := *(*io.ReadSeeker)(opaque)

    fmt.Printf("REQUESTED TO SEEK: whence: %v offset: %v\n", int(whence), int64(offset))

    if whence == CAVSEEK_SIZE || whence == CAVSEEK_FORCE{
        return -1
    }

    n, err := r.Seek(int64(offset), int(whence))
    if err != nil {
        if errors.Is(err, io.EOF) {
            return -1
        }
        fmt.Printf("error seeking: %v\n", err)
    }

    return Cint64_t(n)
}

And here is the output (truncated). It seeks to every single moof box.

REQUESTED TO READ: 4196
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 4196
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 4196
REQUESTED TO SEEK: whence: 0 offset: 615080
REQUESTED TO READ: 4196
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 619276
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 619276
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 619276
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 619276
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 619276
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 619276
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 619276
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 619276
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 619276
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 619276
REQUESTED TO SEEK: whence: 0 offset: 1232664
REQUESTED TO READ: 4196
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 1236860
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 1236860
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 1236860
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 1236860
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 1236860
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 1236860
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 1236860
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 1236860
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 1236860
REQUESTED TO SEEK: whence: 65536 offset: 0
REQUESTED TO SEEK: whence: 2 offset: -1
REQUESTED TO SEEK: whence: 0 offset: 1236860
REQUESTED TO SEEK: whence: 0 offset: 1845238
REQUESTED TO READ: 4196
...