Piping an axios request to a pkgcloud storage upload causes a "write after end" error

Question

I'm trying to download an external file with axios, and then pipe it into an openstack object storage container using pkgcloud. The upload stream appears to end before the file is fully downloaded.

async function upload(url, name, extension, container) {
    const uploadStream = storage.createClient({
        ...
    }).upload({
        container: container,
        remote: name + "." + extension,
    });

    await new Promise(async (resolve) => {
        const response = await axios({
            method: "GET",
            url: url,
            responseType: "stream"
        })

        response.data.pipe(uploadStream)

        response.data.on("end", () => {
            console.log("Download finished");
        })
        response.data.on("error", (error: Error) => {
            console.log("Download error")
            console.log(error);
        })
        uploadStream.on("finish", () => {
            console.log("Upload finished");
            resolve();
        })
        uploadStream.on("error", (error: Error) => {
            console.log("Upload error");
            console.log(error);
        })
    })
}

const url = "https://picsum.photos/id/566/600/600";

(async () => {
    await upload(url, "mountain", "jpg", "dummy_container")
    console.log("Promise resolved");
})()

The order of events according to console output is as follows:

Download finished
Upload finished
Promise resolved
Upload error
Error: write after end
...

Answer 1

The issue can be worked around by using the concat-stream package to stream the download into a buffer, and then stream the buffer into the upload:

function upload(url, name, extension, container) {
    const uploadStream = storage.createClient({
        ...
    }).upload({
        container,
        remote: name + "." + extension,
    });

    axios({
        method: "GET",
        url,
        responseType: "stream",
    }).then((response) => {
        const concatStream = concat((buffer) => {
            const bufferStream = new Stream.PassThrough();
            bufferStream.end(buffer);
            bufferStream.pipe(uploadStream);
        })
        response.data.pipe(concatStream);
    })
}

const url = "https://picsum.photos/id/566/600/600";
upload(url, "mountain", "jpg", "dummy_container")

Answer 2

The problem comes from a bug in pkgcloud in how it streams files for OpenStack Storage. It's explained in https://stackoverflow.com/a/60787524/4685491