Axios get request failing after second try

Question

I'm trying to make recursive requests to an API that uses a page system. The first request will always go through, however the second request always fails with a 404. If I restart the application, it works perfectly the first time, and again fails the second. It's also not an issue with the second page, even if I request the same page twice, it fails on the second try. If I request the second page first, and the first page second. It fails on the second attempt. All the requests work perfectly through postman.

I've tried using axios, unirest. request and native (all the code snippets from postman).

var axios = require('axios');


const runScrape = () => {
    return new Promise(((resolve, reject) => {
        const api = axios.create()
        var config = {
            method: 'get',
            url: 'https://foo.bar/?page=1',
        };

        api(config)
            .then(function (response) {
                resolve("worked")
            })
            .catch(function (error) {
                reject("failed")
            });

    }))
}

runScrape().then(d => {
    console.log(d)
    runScrape().then(d => {
        console.log(d)
    }).catch(e => console.log(e))
}).catch(e => console.log(e))

Any help would be greatly appreciated!

Answer 1

I fixed the issue by changing the TLS fingerprint. The code looks as followed now.

const axios = require('axios');
const https = require('https')

const httpsAgent = new https.Agent({
    minVersion: "TLSv1.3"
})

const runScrape = () => {
    return new Promise(((resolve, reject) => {
        const config = {
            params: {
                page: "1"
            },
            httpsAgent
        };

        axios.get('https://foo.bar/', config)
            .then(function (response) {
                resolve("worked")
            })
            .catch(function (error) {
                reject("failed")
            });

    }))
}

runScrape().then(d => {
    console.log(d)
    runScrape().then(d => {
        console.log(d)
    }).catch(e => console.log(e))
}).catch(e => console.log(e))