Answer by ccprog for NodeJS Streaming and Request Module

Your attempts start with spliting the full response data into lines with data.split('\n'), which means the response has to be read twice, once for line-splitting, and once for data point parsing.

You should pipe the response to readline:

const request = require('request');const readline = require('readline');function streamPrice = function(req,res){    const { url } = req.query;    function transformLine (rawdata) {        const elem;        // parse        res.write(JSON.stringify(elem) +"\n\n");    }    request.get(url)        // response is an instance of http.IncomingMessage, a readable stream        .on('response', function(response) {            readline.createInterface({                input: response            }).on('line', transformLine);    });}

Your individual lines are not that long, so JSON.stringify should be no problem.

Since you say identifying datasets may be more involved than reading lines, you could implement your own Transform stream:

class ResponseToJSON extends Transform {  constructor (options) {    this.cache = '';    super(options);  }  _transform (data, encoding, callback) {    const chunk = data.toString(encoding);    // get all yet complete lines    for (let line of this.splitData(chunk) {      this.transformLine(line);    }    callback();  }  splitData (chunk) {    //prepend the chunk with the cache content    chunk = this.cache + chunk;    // apply logic to identify single dataset endings    // and write result into an array    const lines = ...    // the last line may be yet incomplete    this.cache = lines.pop();    return lines;  }  transformLine (rawData) {    const elem;    // parse    // push out to receiving stream    this.push(line);  }  _flush (callback) {    // make sure the last line is transformed    this.transformLine(this.cache);    callback();  }}function streamPrice = function(req, res){  const { url } = req.query;  const transform = new ResponseToJSON();  transform.pipe(res);  request.get(url).on('response', function(response) {    response.pipe(transform);  }}

This solution may slow things down agin a bit, since the line splitting again works on strings. If you see a way to identify datasets directly on the buffer, this would be certainly more efficient.