ETIMEDOUT when making large(10k) post requests using axios.all nodejs

Question

I am using axios.all to hit all my post calls at a time. Here is my code

let postUrls = [];
data.forEach(inventory => {
            const userData = {
              stream: stream_name,
              key: inventory.serialNumber,
              address: address,
              data: inventory,
            };
            const postRequest = axios.post(`${serviceURL}/publishExcelData`,
              userData,);
            postUrls.push(postRequest)
          });

 axios.all(postUrls)
                 .then(
                   axios.spread((...responses) => {
                     
                     const inventoryData = responses.map(response => response.data);
                     // use/access the results
                     console.log(inventoryData.length);
                     
                   })
                 )
                 .catch(errors => {
                   // react on errors.
                   console.error(errors);
                 });*/

This works fine with data size of 1000. But when the data size is 1000 then i am getting etimedout error. Any solution for this please. I have tried setting http agentalive true but no use.

Answer 1

I resolved it by making 50 requests at a time till the total count(50k). If any one is facing issue using axios.all here is my code.

let start = new Date();

          let count = 0;
          const chunkSize = 50;
          let limit = chunkSize;
          let skip = 0;
          console.log('Inserting excel data in chunks');
          function recursiveFun() {
            skip = chunkSize * count;
            count++;
            limit = chunkSize * count;
            console.log('skip', skip);

            console.log('limit', limit);
            const chunkedData = data.slice(skip, limit);
            let chunkUrls = [];
            chunkedData.forEach(inventory => {
              const userData = {
                stream: stream_name,
                key: inventory.serialNumber,
                address: address,
                data: inventory,
              };
              const postRequest = axios.post(
                `${blockchain_service_url}/publishExcelData`,
                userData,
              );
              chunkUrls.push(postRequest);
            });
            axios
              .all(chunkUrls)
              .then(
                axios.spread(async (...responses) => {
                  const inventoryData = responses.map(
                    response => response.data,
                  );
                  // use/access the results
                  console.log(inventoryData.length);
                  console.log(inventoryData[0].transactionId);
                
                    InventoryModel.insertMany(inventoryData, (err, res) => {
                      if (err) {
                        console.log(err.errmsg);
                      } else
                        console.log(
                          'Number of documents inserted into mongo: ' +
                            res.length,
                        );
                    });
                 
                  if (limit !== data.length) {
                    recursiveFun();
                  } else {
                    console.log(
                      'Insertion of excel sheet data is completed. Time Taken in seconds - ',
                      (new Date() - start) / 1000,
                    );//Insertion of excel sheet data is completed. Time Taken to post and insert 50k records in seconds -  585.385
                  }
                }),
              )
              .catch(errors => {
                // react on errors.
                console.error(errors);
              });
          }
          recursiveFun();