ioSocket not emitting to browser on server heavy load

Question

Some requests that can be made to my nodejs server require heavy processing (for example: 5000 files). Since the request will take a while to process, I want to display progresses in the browser. For that, I'm using io-socket. Regularly, the server sends the progress advance to the client with for example ioSocket.emit("log", "progress 24%).
However, if progress sending to the client usually works, when there is a big amount of files, it doesn't. Nothing is sent to browser.
I'm sure the process is going fine since I log progress to node terminal, and there it appears as expected.

I'm wondering what I can do to have ioSocket.emit event work in heavy load case, because it's where it's most useful to see progress.

The files processing function looks like this:

var child_process = require("child_process");

function (ioSocket) {

  ioSocket.emit("log", "start")

  var ratingCounts = 0;
  var listOfFilesRatings = []

  _.each(listOfFilesPaths, function(path, i){

    child_process.exec("exiftool -b -Rating "+ path, function(err, stdout){

      if (err) console.log(err)
      else {

        listOfFilesRatings.push(stdout);
        ratingCounts++;

        ioSocket.emit("log", "rating test progress "+ ratingCounts)

      };

    });

    ioSocket.emit("log", "each progress "+ i)

  });


}

In this example, only the first "start" emit will be fired to the browser.

However if I do the following:

function (ioSocket) {

  ioSocket.emit("log", "start")

  for (i=0; i<1000; i++) {

    ioSocket.emit("log", "each progress "+ i)

  };

}

everything works fine, and I get the "start" and all "each progress" sent to browser.

Answer 1

If you are processing 5000 files, your scheme with _.each() and child_process.exec() will launch 5000 exiftool processes at once. That will likely bring any computer, except for perhaps some big iron to its knees. You should probably be launching no more than N of those at a time where you run some performance tests on your particular hardware to determine what N should be (probably under 10).

Here's one way to do that:

var child_process = require("child_process");

function processFiles(ioSocket) {

    return new Promise((resolve, reject) => {

        ioSocket.emit("log", "start")

        let ratingCounts = 0;
        let listOfFilesRatings = [];
        const maxInFlight = 10;
        let inFlightCntr = 0;
        let fileIndex = 0;

        function run() {
            // while room to run more, run them
            while (inFlightCntr < maxInFlight && fileIndex < listOfFilesPaths.length) {
                let index = fileIndex++;
                ++inFlightCntr;
                ioSocket.emit("log", "each progress " + index)
                child_process.exec("exiftool -b -Rating " + path, function(err, stdout) {
                    ++ratingCounts;
                    --inFlightCntr;
                    if (err) {
                        console.log(err);
                        listOfFilesRatings[index] = 0;
                    } else {
                        listOfFilesRatings[index] = stdout;
                        ioSocket.emit("log", "rating test progress " + ratingCounts)
                    }
                    run();
                });
            }
            if (inFlightCntr === 0 && fileIndex >= listOfFilesPaths.length) {
                // all done here
                console.log(listOfFilesRatings);
                resolve(listOfFilesRatings);
            }
        }
        run();
    });
}


processFiles().then(results => {
    console.log(results);
});