NodeJS/Mongo: Looping a query through various collections

Question

I'm looking to loop a query through various collection with MongoDB using the NodeJS Driver. For this test, I've used the sample code from the 'findOne' docs to insert a bunch of documents in various Collections:

  collection.insertMany([{a:1, b:1}, {a:2, b:2}, {a:3, b:3}], {w:1}, function(err, result) {
    test.equal(null, err);

Creating at the same time various collections (each collection has at least one instance of the documents previously inserted):

• test
• test1
• test2
• test3
• test4
• test6
• test10

And what I want is to gather the list of collection that I have in the DB ('test' in my case):

var MongoClient = require("mongodb").MongoClient,
  test = require("assert");
MongoClient.connect("mongodb://localhost:27017/test", function(err, db) {
  db.listCollections().toArray(function(err, items) {
    test.ok(items.length >= 1);
    console.log(items);
    db.close();
  });
});

And there pops the list of collection previously mentioned. Up until now everything is all-right! I can even loop through the array to get the name of the collections only:

var MongoClient = require("mongodb").MongoClient,
  test = require("assert");
MongoClient.connect("mongodb://localhost:27017/test", function(err, db) {
  db.listCollections().toArray(function(err, items) {
    test.ok(items.length >= 1);
    items.forEach(c => {
      console.log(c.name);
    });
    db.close();
  });
});

Again no problem there! But when I then try a query within the loop:

var MongoClient = require("mongodb").MongoClient,
  test = require("assert");
MongoClient.connect("mongodb://localhost:27017/test", function(err, db) {
  db.listCollections().toArray(function(err, items) {
    test.ok(items.length >= 1);
    items.forEach(c => {
      var collection = db.collection(c.name);
      collection.findOne({ a: 2 }, { fields: { b: 1 } }, function(err, doc) {
        console.log(doc);
      });
    });
  });
  db.close();
});

I get:

null
null
null
null
null
null
null

Even though looping through to get the collection seems to work perfectly fine:

var MongoClient = require("mongodb").MongoClient,
  test = require("assert");
MongoClient.connect("mongodb://localhost:27017/test", function(err, db) {
  db.listCollections().toArray(function(err, items) {
    test.ok(items.length >= 1);
    items.forEach(c => {
      var collection = db.collection(c.name);
      console.log(collection);
      });
    });
  db.close();
});

Example output:

Collection {
  s: 
   { pkFactory: 
      { [Function: ObjectID]
        index: 10866728,
        createPk: [Function: createPk],
        createFromTime: [Function: createFromTime],
        createFromHexString: [Function: createFromHexString],
        isValid: [Function: isValid],
        ObjectID: [Circular],
        ObjectId: [Circular] },
     db: 
      Db {
        domain: null,
        _events: {},
        _eventsCount: 0,
        _maxListeners: undefined,
        s: [Object],
        serverConfig: [Getter],
        bufferMaxEntries: [Getter],
        databaseName: [Getter] },
     topology: 
      Server {
        domain: null,
        _events: [Object],
        _eventsCount: 8,
        _maxListeners: undefined,
        clientInfo: [Object],
        s: [Object] },
     dbName: 'test',
     options: 
      { promiseLibrary: [Function: Promise],
        readConcern: undefined,
        readPreference: [Object] },
     namespace: 'test.test2',
     readPreference: 
      ReadPreference {
        _type: 'ReadPreference',
        mode: 'primary',
        tags: undefined,
        options: undefined },
     slaveOk: true,
     serializeFunctions: undefined,
     raw: undefined,
     promoteLongs: undefined,
     promoteValues: undefined,
     promoteBuffers: undefined,
     internalHint: null,
     collectionHint: null,
     name: 'test2',
     promiseLibrary: [Function: Promise],
     readConcern: undefined } }

I'm guessing that the Collection structure is the problem for my loop but I'm not sure what's happening exactly... This is an example of the expected output for each Collection:

{ _id: 5a13de85a55e615235f71528, b: 2 }

Any help would be much appreciated! Thanks in advance!

Answer 1

Though not the best syntax and useless except for logging output, this does work for me:

var mongodb = require('mongodb');


mongodb.connect('mongodb://localhost:27017/test', function (err, db) {
    if (err) {
        throw err;
    }

    db.listCollections().toArray(function (err, cols) {
        if (err) {
            throw err;
        }

        cols.forEach(function (col) {
            db.collection(col.name).find({}, {}, 0, 1, function (err, docs) {
                if(err){
                    throw err;
                }
                console.log(col);
                docs.forEach(console.log);
            });
        });
    })
})

So, perhaps the query conditions don't match anything ?

Also, better with Promises:

const mongodb = require('mongodb');
const Promise = require('bluebird');

function getDb() {
    return Promise.resolve(mongodb.connect('mongodb://localhost:27017/test'));
}

function getCollections(db) {
    return Promise.resolve(db.listCollections().toArray());
}

function getDocs(db, col) {
    return Promise.resolve(db.collection(col.name).find({},{},0,1).toArray());
}

const data = {};
getDb()
.then((db) => {
    data.db = db;
    return getCollections(db);
}).then((cols) => {
    data.cols = cols;
    return Promise.map(cols, (col) => getDocs(data.db,col));
}).then((docs) => {
    console.log(docs);
})

Answer 2

ForEach loop in javaScript is synchronous until it doesn't contain any asynchronous call. You cannot call any DB call within for loop in this case.

Instead, you can use a library named async which comes with some awesome functions to take care of this problem as below

var MongoClient = require("mongodb").MongoClient,
    test = require("assert"),
    async = require('async');
MongoClient.connect("mongodb://localhost:27017/test", function (err, db) {
db.listCollections().toArray(function (err, items) {
    test.ok(items.length >= 1);

    async.map(items, (each, callback) => {
        let collection = db.collection(each.name);
        collection.findOne({a: 2}, {fields: {b: 1}}, function (err, doc) {
            console.log(doc);
            callback();
        });
    }, (err) => {
        console.log("done");
    });
  });
  db.close();
});