Node.js: What techniques are there for writing clean, simple callback code?

Question

node.js code is known for turning into callback spaghetti.

What are the best techniques for overcoming this problem and writing clean, uncomplex, easy to understand

Answer 1

Several things can be done to avoid the 'matrioska-style'.

• You can store callbacks to variables:

  var on_read = function (foo, bar) {
        // some logic 
      },
  
      on_insert = function (err, data) {
        someAsyncRead(data, on_read);
      };
  
  someAsyncInsert('foo', on_insert);
  

• You can use some modules that help in those scenarios.

  // Example using funk
  var funk = require('funk');
  for(var i = 0; i < 10; i++) {
    asyncFunction(i, funk.add(function (data) {
      this[i] = data;
    }));
  }
  
  funk.parallel(function () {
    console.log(this);
  });
  

Answer 2

For the most part, working Twitter OAuth2 application-only example, using Kris' Q promise library with https.request, Nodejs Express api route. First attempt user timeline GET. If 401 response, refreshing bearer-token then retry user timeline. I had to use Q.when to handle a promise that returns another promise (chaining) or a value.

 /**
 * Using Rails-like standard naming convention for endpoints.
 * GET     /things              ->  index
 * POST    /things              ->  create
 * GET     /things/:id          ->  show
 * PUT     /things/:id          ->  update
 * DELETE  /things/:id          ->  destroy
 */

'use strict';

// var _ = require('lodash');
var http = require('http');
var https = require('https');
var querystring = require('querystring');
var Q = require('q')

// Get list of twtimelines
exports.index = function(req, res) {
    var tid = req.query.tid
    if (tid) {
        Q.when(reqTimeline(tid, true, res), function(value) {
            // > value
            // 404
            // > body1
            // '{"errors":[{"code":34,"message":"Sorry, that page does not exist."}]}'
        })
    } else {
        res.json({
            errors: [{
                message: 'no tid specified in query'
            }]
        });
    }
};


function reqPromise(options, postData) {
    var deferred = Q.defer()

    var req = https.request(options, function(res) {
        // console.log("statusCode: ", res.statusCode);
        // console.log("headers: ", res.headers);
        var statusCode = res.statusCode
        deferred.notify(res)

        res.on('data', function(d) {
            //process.stdout.write(d);
            deferred.notify(d)
        }).on('end', function() {
            deferred.resolve(statusCode)
        });
    });

    req.on('error', function(e) {
        console.error(e);
        deferred.reject(e)
    });

    req.write(postData);
    req.end();
    return deferred.promise
} // deferRequest

function isIncomingMessage(ot) {
    return ot instanceof http.IncomingMessage
}

function isBuffer(ot) {
    return ot instanceof Buffer
}

function reqTimeline(screen_name, reqBearerTokenOn401, res) {
    var optionsUserTimeline = {
        hostname: 'api.twitter.com',
        path: '/1.1/statuses/user_timeline.json?' + querystring.stringify({
            count: '3',
            screen_name: screen_name
        }),
        method: 'GET',
        headers: {
            //'Authorization': 'Bearer ' + JSON.parse(body1).access_token
            'Authorization': 'Bearer ' + process.env.BEARER_TOKEN
        } // headers
    };
    console.log("optionsUserTimeline", optionsUserTimeline)

    var statusCode;
    var body1 = new Buffer(''); // default utf8 string buffer ?
    return reqPromise(optionsUserTimeline, '')
        .then(function(value) { // done
                if (reqBearerTokenOn401 && value === 401) {
                    console.log("reqTimeline - requesting bearer token")
                    return reqBearerToken(screen_name, res)
                }
                console.log("reqTimeline - done done:", value)
                res.end()
                return value
            },
            function(reason) { // error
                console.log("reqTimeline - error:", body1)
            },
            function(progress) {
                console.log("reqTimeline - progress:", body1)
                if (isIncomingMessage(progress)) {
                    body1 = body1.slice(0, 0) // re-set buffer
                    statusCode = progress.statusCode;
                    if (reqBearerTokenOn401 && statusCode === 401) {
                        // readyn for retry
                    } else {
                        res.writeHead(statusCode)
                    }
                } else if (isBuffer(progress)) {
                    if (reqBearerTokenOn401 && statusCode === 401) {
                        body1 += progress
                    } else {
                        res.write(progress)
                    }
                } else {
                    throw "reqTimeline - unexpected progress"
                }
            });
} // reqTimeline

function reqBearerToken(screen_name, res) {
    var postData = querystring.stringify({
        'grant_type': 'client_credentials'
    })
    var optionsBearerToken = {
            hostname: 'api.twitter.com',
            path: '/oauth2/token',
            method: 'POST',
            headers: {
                'Authorization': 'Basic ' + new Buffer(
                    process.env.CONSUMER_KEY + ":" + process.env.CONSUMER_SECRET
                ).toString('base64'),
                'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8',
                'Content-Length': postData.length
            } // headers
        }
        // console.log("key", process.env.CONSUMER_KEY)
        // console.log("secret", process.env.CONSUMER_SECRET)
        // console.log("buf", new Buffer(
        //  process.env.CONSUMER_KEY + ":" + process.env.CONSUMER_SECRET
        // ).toString())
        console.log("optionsBearerToken", optionsBearerToken)

    var body2 = new Buffer(''); // default utf8 string buffer ?
    return reqPromise(optionsBearerToken, postData)
        .then(function(value) { // done
            console.log("reqBearerToken - done:", body2)
            if (value === 200) {
                console.log("reqBearerToken - done done")
                process.env.BEARER_TOKEN = JSON.parse(body2).access_token;
                return reqTimeline(screen_name, false, res)
            }
            return value
        }, function(reason) {
            throw "reqBearerToken - " + reason
        }, function(progress) {
            if (isIncomingMessage(progress)) {
                body2 = body2.slice(0, 0) // reset buffer
            } else if (isBuffer) {
                body2 += progress
            } else {
                throw "reqBearerToken - unexpected progress"
            }
        });
} // reqBearerToken

Answer 3

Take a look at Promises: http://promises-aplus.github.io/promises-spec/

It is an open standard which intended to solve this issue.

I am using node module 'q', which implements this standard: https://github.com/kriskowal/q

Simple use case:

var Q = require('q');

For example we have method like:

var foo = function(id) {
  var qdef = Q.defer();

  Model.find(id).success(function(result) {
    qdef.resolve(result);
  });

  return (qdef.promise);
}

Then we can chain promises by method .then():

foo(<any-id>)
.then(function(result) {
  // another promise
})
.then(function() {
  // so on
});

It is also possible to creating promise from values like:

Q([]).then(function(val) { val.push('foo') });

And much more, see docs.

See also:

• http://jeditoolkit.com/2012/04/26/code-logic-not-mechanics.html#post
• http://wiki.commonjs.org/wiki/Promises/A

Answer 4

Try node-line

https://github.com/kevin0571/node-line

Usage:

var line = require("line");
line(function(next) {
    obj.action1(param1, function(err, rs) {
        next({
            err: err,
            rs: rs
        });
    });
}, function(next, data) {
    if (data.err) {
        console.error(err);
        return;
    }
    obj.action2(param2, function(err, rs) {
        if (err) {
            console.error(err);
            return;
        }
        next(rs);
   });
}, function(rs) {
   obj.finish(rs);
});

Answer 5

I'd suggest 1) using CoffeeScript and 2) using named callbacks and passing state between them in a hash, rather than either nesting callbacks or allowing argument lists to get very long. So instead of

var callback1 = function(foo) {
  var callback2 = function(bar) {
    var callback3 = function(baz) {
      doLastThing(foo, bar, baz);
    }
    doSomethingElse(bar, callback3);
  }
  doSomething(foo, callback2);
}
someAsync(callback1);

you can instead simply write

callback1 = (state) -> doSomething state.foo, callback2
callback2 = (state) -> doSomethingElse state.bar, callback3
callback3 = (state) -> doLastThing state
someAsync callback1

once your doSomething, doSomethingElse and doLastThing have been rewritten to use/extend a hash. (You may need to write extra wrappers around external functions.)

As you can see, the code in this approach reads neatly and linearly. And because all callbacks are exposed, unit testing becomes much easier.