nodejs createWriteStream in nested for

Question

I need to generate a huge number of combinations of string values, a few billion of lines and write them all in one large file to disk without the process being blocked.

At this moment it works until the file write buffer is filled, for little lists of words, when I try with a larger number of records the output files remains empty.

const fs = require('fs');

const words = fs.readFileSync('./1024words.txt').toString().split("\n");
//create array of 1024 words

let fileout
    , row = ''
    , wordA, wordB, wordC
    , totRows = Math.pow(words.length,3)*3  //combination of 3 words for 3 columns of output
    , count = 0;

for (let indexA = 0; indexA < words.length; indexA++) {
    wordA = words[ indexA ];

    fileout = fs.createWriteStream(`${wordA}.csv`); //new output file

    for (let indexB = 0; indexB < words.length; indexB++) {
        wordB = words[ indexB ];

        for (let indexC = 0; indexC < words.length; indexC++) {
            wordC = words[ indexC ];

            row = `${wordA} ${wordB} ${wordC}\n`;

            count++;

            process.stdout.write(`${count} of ${totRows} written: ${row}`);

            fileout.write(row);
        }
    }

    fileout.end();
    fileout.close();
}

Answer 1

you're missing a backpressure handling

further reading here

https://nodejs.org/en/docs/guides/backpressuring-in-streams/

add this function

 function writeToStream(stream, data) {
        const canContinue = stream.write(data);
        if (!canContinue) {
            stream.pause()
            stream.once('drain', () => {
                stream.resume()
            })
        }
    }

full code :

const fs = require('fs');

const words = fs.readFileSync('./1024words.txt').toString().split("\n");
//create array of 1024 words

let fileout
    , row = ''
    , wordA, wordB, wordC
    , totRows = Math.pow(words.length, 3) * 3  //combination of 3 words for 3 columns of output
    , count = 0;

for (let indexA = 0; indexA < words.length; indexA++) {
    wordA = words[indexA];

    fileout = fs.createWriteStream(`${wordA}.csv`); //new output file

    for (let indexB = 0; indexB < words.length; indexB++) {
        wordB = words[indexB];

        for (let indexC = 0; indexC < words.length; indexC++) {
            wordC = words[indexC];

            row = `${wordA} ${wordB} ${wordC}\n`;

            count++;
            writeToStream(process.stdout, `${count} of ${totRows} written: ${row}`)
            writeToStream(fileout, row)
        }
    }

    fileout.end();
    fileout.close();
}

function writeToStream(stream, data) {
    const canContinue = stream.write(data);
    if (!canContinue) {
        stream.pause()
        stream.once('drain', () => {
            stream.resume()
        })
    }
}