NodeJS -> GET data string after each res.write(data)

Question

I have two NodeJS applications running on localhost.

Application no.1 requests /generatedData form application no.2 (below) using superagent.js:

request = require('superagent');
request.get('http://localhost:3000/generatedData')
.end(function(err, res) {
    if (err) {
        console.log(err);
    } else {
        console.log(res.text);
    }
});

Application no.2 generates the data and writes it to the response (below)

router.get('/generatedData', function (req, res) {

    res.setHeader('Connection'          , 'Transfer-Encoding');
    res.setHeader('Content-Type'        , 'text/html; charset=utf-8');
    res.setHeader('Transfer-Encoding'   , 'chunked');

    var Client = someModule.client;  
    var client = Client();

    client.on('start', function() {  
        console.log('start');
    });

    client.on('data', function(data) {
        res.write(data);
    });

    client.on('end', function(msg) { 
        client.stop();
        res.end();
    });

    client.on('err', function(err) { 
        client.stop();
        res.end(err);
    });

    client.on('stop', function() {  
        console.log('stop');
    });

    client.start();

    return;

});

In app no.1 i would like to use the data as it being written. I cant wait till request.end as the data being generated is potentially huge and will take a long time to finish.

How can i get the data as it's being written to the response by app no.2?

Is this the right direction? what's the best way to do this?

Thanks, Asaf

Answer 1

Streaming is missing from the first app.

You could use something:

require('superagent')
  .get('www.streaming.example.com')
  .type('text/html')
  .end().req.on('response',function(res){
      res.on('data',function(chunk){
          console.log(chunk)
      })
      res.pipe(process.stdout)
  })

Take reference form Streaming data events aren't registered.

Use something like this if you want to write to the file...

const request = require('superagent');
const fs = require('fs');

const stream = fs.createWriteStream('path/to/my.json');

const req = request.get('/some.json');
req.pipe(stream);

Answer 2

To use data as it being written in App No.1, you can use Node.js http module and listen data event of response object.

const http = require('http');

const req = http.request({
  hostname: 'localhost',
  port: 3000,
  path: '/generatedData',
  method: 'GET'
}, function(res) {
  res.on('data', function(chunk) {
    console.log(chunk.toString());
    // do whatever you want with chunk
  });
  res.on('end', function() {
    console.log('request completed.');
  });
});

req.end();