Decompress streaming response using pipe syntax

Question

I am attempting to retrieve a file.csv.gz using request.js, and decompress it, before parsing it and processing.

I know I want my code to look like this

response
.pipe(zlib.createGunzip())
.pipe(split())
.pipe(process())

however, I am struggling to get my response object in the correct format to be piped. I am currently attempting to stream the response by doing;

const request = require('request');

const headers = {
  'accept-encoding':'gzip'
}

module.exports.getCSV = (url) => {
  return request({url, headers, gzip:true});
}

I am receiving errors which imply that I am attempting to decompress an incomplete object.

I am also thinking that it is perhaps not possible to do what I want to achieve, and instead I will need to fully download the file, before attempting to parse it for processing

Answer 1

receive.js

const http = require('http');
const fs = require('fs');
const zlib = require('zlib');

const server = http.createServer((req, res) => {
    const filename = req.headers.filename;
    console.log('File request received: ' + filename);
    req
     .pipe(zlib.createGunzip())
     .pipe(fs.createWriteStream(filename))
     .on('finish', () => {
        res.writeHead(201, {'Content-Type': 'text/plain'});
        res.end('Complete\n');
        console.log(`File saved: ${filename}`);
     });
 });
 server.listen(3000, () => console.log('Listening'));

send.js

const fs = require('fs');
const zlib = require('zlib');
const http = require('http');
const path = require('path');
const file = 'myfile.txt'; //Put your file here
const server = 'localhost'; //Put the server here
const options = {
    hostname: server,
    port: 3000,
    path: '/',
    method: 'PUT',
    headers: {
      filename: path.basename(file),
      'Content-Type': 'application/octet-stream',
      'Content-Encoding': 'gzip'
      }
  };

const req = http.request(options, res => {
   console.log('Server response: ' + res.statusCode);
});
fs.createReadStream(file)
 .pipe(zlib.createGzip())
 .pipe(req)
 .on('finish', () => {
   console.log('File successfully sent');
  });