Stream read returning empty in nodejs

Question

I am using express to create a webservice that will read string data from a stream, and respond to the HTTP POST request with that value. Here is the code for the S3Store.js file that defines the readFileFromS3(.) function:

S3Store.js

S3Store.prototype.readFileFromS3 = function(filename, callback) {
var readConfig = {
    'Bucket': 'shubham-test',
    'Key': filename
};
var readStream = this.s3.getObject(readConfig).createReadStream();
var allData = '';

readStream.on('data', function(data) {
    //data = Buffer.concat([allData, data]);
    data = allData + data;
    console.log("data: " + data);
});

readStream.on('error', function(err) {
    callback(err, null);
});

Now, if I call this method from a terminal like this:

s3Instance.readFileFromS3('123.json', function(err, data) {
    console.log(data);
});

I see the appropriate string for data logged to the console. However, when I call the same method from inside one of the routes in express for HTTP POST requests, the service responds with a value of data set to empty string. Code for the POST request:

router.post('/resolve', function(req, res) {
    var commandJson = req.body;

    var appId = commandJson['appId'];
    var command = commandJson['text'];

    if (appId == undefined || command == undefined) {
        res.status(400).send("Malformed Request: appId: " + appId + ", command: " + command);
    };

    s3Store.readFileFromS3('123.json', function(err, data) {
        res.send(data);
    });
});

Why does it return an empty string when calling the readFileFromS3(.) from the HTTP POST method and not when I ran the same method directly from the node console?

Answer 1

You're logging the data but you're not passing anything to the completion callback (see below for some more explanation):

S3Store.prototype.readFileFromS3 = function(filename, callback) {
  var readConfig = {
      'Bucket': 'shubham-test',
      'Key': filename
  };
  var readStream = this.s3.getObject(readConfig).createReadStream();
  var allData = [];

  // Keep collecting data.
  readStream.on('data', function(data) {
    allData.push(data);
  });

  // Done reading, concatenate and pass to completion callback.
  readStream.on('end', function() {
    callback(null, Buffer.concat(allData));
  });

  // Handle any stream errors.
  readStream.on('error', function(err) {
    callback(err, null);
  });
};

I took the liberty to rewrite the data collection to use Buffer's instead of strings, but this obviously isn't a requirement.

The callback argument is a completion function, meant to be called when either reading the S3 stream is done, or when it has thrown an error. The error handling was already in place, but not the part where you would call back when all the data from the stream was read, which is why I added the end handler.

At that point, the readStream is exhausted (everything from it has been read into allData), and you call the completion callback when the collected data as second argument.

The common idiom throughout Node is that completion callbacks take (at least) two arguments: the first is either an error, or null when there aren't errors, and the second is the data you want to pass back to the caller (in your case, the anonymous function in your route handler that calls res.send()).