Node.js, the pyramid of doom (even with async), can you write it better?

Question

I consider myself a very experienced node.js developer.

Yet I still wonder if there is a better way to write the following code so I don\'t get the pyramid of doom..

Answer 1

The simplest way to combat the async pyramid of hell is to segregate your async callbacks into smaller functions that you can place outside your main loop. Chances are you can at least break some of your callbacks into more maintainable functions that can be used elsewhere in your code base, but the question you're asking is a bit vague and can be solved in a large number of ways.

Also, you should consider what Stuart mentioned in his answer and try to combine some of your queries together. I'm more concerned that you have 20+ nested calls which would indicate something seriously erroneous in your callback structure so I'd look at your code first before anything else.

Answer 2

The problem to this sort of thing is promises. If you haven't heard of them, I suggest reading up on kriskowal's q.

Now, I don't know if the db.query returns a promise or not. If it doesn't you might be able to find a db-wrapper that does or a different db library. If that is not an option, you may "promisify" the db-library you're using. See Howto use promises with Node, and especially the section "Wrapping a function that takes a Node-style callback".

Best of luck! :)

Answer 3

Consider rewriting your code to have less back-and-forth with the database. The rule of thumb I use to estimate an app's performance under heavy load is that every async call will add two seconds to the response (one for the request, and one for the reply).

For example, is there maybe a way you could offload this logic to the database? Or a way to "SELECT nextval('last_resource_bundle_string_id')" at the same time as you "SELECT * FROM localization_strings WHERE local_id = 10 AND string_key = '" + key[0] + "'" (perhaps a stored procedure)?

Answer 4

I break each level of the pyramid of doom into a function and chain them one to the other. I think it is a lot easier to follow. In the example above i'd do it as follows.

ms.async.eachSeries(arrWords, function (key, asyncCallback) {

  var pgCB;
  var pgClient;

  var connect = function () {
    pg.connect(pgconn.dbserver('galaxy'), function (err, _pgClient, _pgCB) {
      pgClient = _pgClient;
      pgCB = _pgCB;
      findKey();
    });
  };

  var findKey = function () {
    statement = "SELECT * FROM localization_strings WHERE local_id = 10 AND string_key = '" + key[0] + "'";
    pgClient.query(statement, function (err, result) {
      if (pgconn.handleError(err, pgCB, pgClient))
        return;
      // if key doesn't exist go ahead and insert it
      if (result.rows.length == 0) {
        getId();
        return;
      }
      pgCB(); 
      asyncCallback(); 
    });

  };

  var getId = function () {
    statement = "SELECT nextval('last_resource_bundle_string_id')";
    pgClient.query(statement, function (err, result) {
      if (pgconn.handleError(err, pgCB, pgClient))
        return;
      insertKey();
    });
  };

  var insertKey = function () {
    var insertIdOffset = parseInt(result.rows[0].nextval);
    statement = "INSERT INTO localization_strings (resource_bundle_string_id, string_key, string_revision, string_text,modified_date,local_id, bundle_id) VALUES ";
    statement += "  (" + insertIdOffset + ",'" + key[0] + "'," + 0 + ",'" + englishDictionary[key[0]] + "'," + 0 + ",10,20)";
    ms.log(statement);
    pgClient.query(statement, function (err, result) {
      if (pgconn.handleError(err, pgCB, pgClient))
        return;
      pgCB();
      asyncCallback();
    });
  };

  connect();

});

Answer 5

As Mithon said in his answer, promises can make this code much clearer and help to reduce duplication. Let's say that you create two wrapper functions that return promises, corresponding to the two database operations you're performing, connectToDb and queryDb. Then your code can be written as something like:

ms.async.eachSeries(arrWords, function (key, asyncCallback) {
  var stepState = {};
  connectToDb('galaxy').then(function(connection) {
    // Store the connection objects in stepState
    stepState.pgClient = connection.pgClient;
    stepState.pgCB = connection.pgCB;

    // Send our first query across the connection
    var statement = "SELECT * FROM localization_strings WHERE local_id = 10 AND string_key = '" + key[0] + "'";
    return queryDb(stepState.pgClient, statement);
  }).then(function (result) {
    // If the result is empty, we need to send another 2-query sequence
    if (result.rows.length == 0) {
       var statement = "SELECT nextval('last_resource_bundle_string_id')";
       return queryDb(stepState.pgClient, statement).then(function(result) {
         var insertIdOffset = parseInt(result.rows[0].nextval);
         var statement = "INSERT INTO localization_strings (resource_bundle_string_id, string_key, string_revision, string_text,modified_date,local_id, bundle_id) VALUES ";
         statement += "  (" + insertIdOffset + ",'" + key[0] + "'," + 0 + ",'" + englishDictionary[key[0]] + "'," + 0 + ",10,20)";
         ms.log(statement);
         return queryDb(stepState.pgClient, statement);
       });
     }
  }).then(function (result) {
    // Continue to the next step
    stepState.pgCB();
    asyncCallback();
  }).fail(function (error) {
    // Handle a database error from any operation in this step...
  });
});

It's still complex, but the complexity is more manageable. Adding a new database operation to every "step" no longer requires a new level of indentation. Also notice that all error handling is done in one place, rather than having to add an if (pgconn.handleError(...)) line every time you perform a database operation.

Update: As requested, here's how you might go about defining the two wrapper functions. I'll assume that you're using kriskowal/q as your promise library:

function connectToDb(dbName) {
  var deferred = Q.defer();
  pg.connect(pgconn.dbserver(dbName), function (err, pgClient, pgCB) {
    if (err) {
      deferred.reject(err)
    } else {
      deferred.resolve({pgClient: pgClient, pgCB: pgCB})
    }
  });
  return deferred.promise;
}

You can use this pattern to create a wrapper around any function that takes a single-use callback.

The queryDb is even more straightforward because its callback gives you either a single error value or a single result value, which means that you can use q's built-in makeNodeResolver utility method to resolve or reject the deferred:

function queryDb(pgClient, statement) {
  var deferred = Q.defer();
  pgClient.query(statement, deferred.makeNodeResolver());
  return deferred.promise;
}

For more information on promises, check out my book: Async JavaScript, published by PragProg.