Reading and processing files does not finish before attempting write to a new file. Promise related

Question

Really struggling here and just lost at this point. Promises SEEM simple until I try to use them. I am providing my simplified code that includes no attempt at a promise to keep things simple. I have come back to this after a few months of just giving up. This little script reads the files in a directory line by line and adds all of those lines to an array. Once that is over, I want to write all of those files to a new file.
The problem, of course, is that writeAllLinesToFile is being called before the allTheLines array is full of lines read from the directories files. The array is completely empty and nothing from the read files are written in the new file. I've given a lot of tries with this and just made a mess of my real code in that attempt. Can someone push me over this hump?

var allTheLines = [];

function processFile(inputFile) {
    var fs = require('fs');
    var readline = require('readline');
    var instream = fs.createReadStream(inputFile);
    var outstream = new (require('stream'))();
    var rl = readline.createInterface(instream, outstream);

    // when you get a line, do this 
    rl.on('line', function (line) {
        // console.log("pushing: ", line);
        allTheLines.push(line);
    });

    // when there are no more lines do this
    rl.on('close', function (line) {
        console.log('...end of a file');
    });
}

function writeAllLinesToFile() {
    var fs = require('fs');
    for (var i = 0; i < allTheLines.length; i++) {
        console.log("line: : ", allTheLines[i]);
        fs.appendFile('./allTheLines.html', allTheLines[i] + '\n', function (err) {
            if (err) throw err;
        });
    };
    console.log("Done writing.");
}

// ***************************************
// Execution Starts Here
// ***************************************
var fs = require('fs');
fs.readdir('./filesToRead', (err, files) => {
    for (var i = 0; i < files.length; i++) {
        processFile('./filesToRead/' + files[i]);
        console.log('file#: ', i);
    };
    console.log('Done processing files.');
    writeAllLinesToFile();
});

Answer 1

Here's my quick solution to this and you can take from here and improve it as you see it fit.

I believe the most important thing you forgot is that when we use event emitters, we should know that they're asynchronous code and not synchronous, that's why writeAllLinesToFile had empty array because it was called before rl.on('line', ....) did its job and it's tricky to catch up ;)

Hope this will help you understand how Promises also work and always think of asynchronous code carefully.

var fs = require('fs');
var readline = require('readline');
var path = require('path');
var stream = require('stream');

function processFile(inputFile) {
    var instream = fs.createReadStream(inputFile);
    var outstream = new stream();
    var rl = readline.createInterface(instream, outstream);
    var allTheLines = [];

    // when you get a line, do this 
    rl.on('line', function (line) {
        allTheLines.push(line);
    }).on('close', function () {
        writeAllLinesToFile(allTheLines);
    });
}

function writeAllLinesToFile(lines) {
    for (var i = 0; i < lines.length; i++) {
        fs.appendFile(path.join(__dirname, 'allTheLines.html'), lines[i] + '\n', function (err) {
            if (err) throw err;
        });
    };
  
    console.log("Done writing.");
}

// ***************************************
// Execution Starts Here
// ***************************************

fs.readdir(path.join(__dirname, 'filesToRead'), (err, files) => {
  if (err) throw err;

    for (var i = 0; i < files.length; i++) {
        processFile(path.join(__dirname, 'filesToRead', files[i]));
    };

    console.log('Done processing files.');
});