How can I save multiple documents concurrently in Mongoose/Node.js?

Question

At the moment I use save to add a single document. Suppose I have an array of documents that I wish to store as single objects. Is there a way of adding them all with a si

Answer 1

Bulk inserts in Mongoose can be done with .insert() unless you need to access middleware.

Model.collection.insert(docs, options, callback)

https://github.com/christkv/node-mongodb-native/blob/master/lib/mongodb/collection.js#L71-91

Answer 2

Use insertMany function to insert many documents. This sends only one operation to the server and Mongoose validates all the documents before hitting the mongo server. By default Mongoose inserts item in the order they exist in the array. If you are ok with not maintaining any order then set ordered:false.

Important - Error handling:

When ordered:true validation and error handling happens in a group means if one fails everything will fail.

When ordered:false validation and error handling happens individually and operation will be continued. Error will be reported back in an array of errors.

Answer 3

Add a file called mongoHelper.js

var MongoClient = require('mongodb').MongoClient;

MongoClient.saveAny = function(data, collection, callback)
{
    if(data instanceof Array)
    {
        saveRecords(data,collection, callback);
    }
    else
    {
        saveRecord(data,collection, callback);
    }
}

function saveRecord(data, collection, callback)
{
    collection.save
    (
        data,
        {w:1},
        function(err, result)
        {
            if(err)
                throw new Error(err);
            callback(result);
        }
    );
}
function saveRecords(data, collection, callback)
{
    save
    (
        data, 
        collection,
        callback
    );
}
function save(data, collection, callback)
{
    collection.save
    (
        data.pop(),
        {w:1},
        function(err, result)
        {
            if(err)
            {               
                throw new Error(err);
            }
            if(data.length > 0)
                save(data, collection, callback);
            else
                callback(result);
        }
    );
}

module.exports = MongoClient;

Then in your code change you requires to

var MongoClient = require("./mongoHelper.js");

Then when it is time to save call (after you have connected and retrieved the collection)

MongoClient.saveAny(data, collection, function(){db.close();});

You can change the error handling to suit your needs, pass back the error in the callback etc.

Answer 4

Use async parallel and your code will look like this:

  async.parallel([obj1.save, obj2.save, obj3.save], callback);

Since the convention is the same in Mongoose as in async (err, callback) you don't need to wrap them in your own callbacks, just add your save calls in an array and you will get a callback when all is finished.

If you use mapLimit you can control how many documents you want to save in parallel. In this example we save 10 documents in parallell until all items are successfully saved.

async.mapLimit(myArray, 10, function(document, next){
  document.save(next);
}, done);

Answer 5

Here is another way without using additional libraries (no error checking included)

function saveAll( callback ){
  var count = 0;
  docs.forEach(function(doc){
      doc.save(function(err){
          count++;
          if( count == docs.length ){
             callback();
          }
      });
  });
}

Answer 6

You can use the promise returned by mongoose save, Promise in mongoose does not have all, but you can add the feature with this module.

Create a module that enhance mongoose promise with all.

var Promise = require("mongoose").Promise;

Promise.all = function(promises) {
  var mainPromise = new Promise();
  if (promises.length == 0) {
    mainPromise.resolve(null, promises);
  }

  var pending = 0;
  promises.forEach(function(p, i) {
    pending++;
    p.then(function(val) {
      promises[i] = val;
      if (--pending === 0) {
        mainPromise.resolve(null, promises);
      }
    }, function(err) {
      mainPromise.reject(err);
    });
  });

  return mainPromise;
}

module.exports = Promise;

Then use it with mongoose:

var Promise = require('./promise')

...

var tasks = [];

for (var i=0; i < docs.length; i++) {
  tasks.push(docs[i].save());
}

Promise.all(tasks)
  .then(function(results) {
    console.log(results);
  }, function (err) {
    console.log(err);
  })