How to limit (or queue) calls to external processes in Node.JS?

Question

Scenario

I have a Node.JS service (written using ExpressJS) that accepts image uploads via DnD (example). After an image is uploaded, I do a few thi

Answer 1

Since Node does not allow threading, you can do work in another process. You can use a background job system, like resque, where you queue up jobs to be handled into a datastore of some type and then run a process (or several processes) that pulls jobs from the datastore and does the processing; or use something like node-worker and queue your jobs into the workers memory. Either way, your main application is freed up from doing all the processing and can focus on serving web requests.

[Update] Another interesting library to check out is hook.io, especially if you like the idea of node-workers but want to run multiple background processes. [/Update]

[Edit]

Here's a quick and dirty example of pushing work that takes a while to run to a worker process using node-worker; the worker queues jobs and processes them one by one:

app.js

var Worker = require('worker').Worker;
var processor = new Worker('image_processor.js');

for(var i = 0; i <= 100; i++) {
  console.log("adding a new job");
  processor.postMessage({job: i});
}

processor.onmessage = function(msg) {
  console.log("worker done with job " + msg.job);
  console.log("result is " + msg.data.result);
};

image_processor.js

var worker = require('worker').worker;
var queue = [];

worker.onmessage = function(msg) {
  var job = msg.job;
  queue.push(job);
}

var process_job = function() {
  if(queue.length == 0) {
    setTimeout(process_job, 100);
    return;
  }

  var job = queue.shift();
  var data = {};

  data.result = job * 10;

  setTimeout(function() {
    worker.postMessage({job: job, data: data});
    process_job();
  }, 1000);
};

process_job();

Answer 2

For anyone who thought Brandon's quick-and-dirty might be too quick-and-dirty, here's a variation that is no longer and doesn't have the unnecessary busy-wait. I'm not in a position to test it but it should work.

var enqueue = function() {
  var queue = [];
  var execImmediate = function(fImmediate) {
    enqueue = function(fDelayed) 
      queue.push(fDelayed);
    };
    fImmediate();

    var ic = setInterval(function() {
      var fQueued = queue.shift();
      if (fQueued) {
        fQueued();
      } else {
        clearInterval(ic);
        enqueue = execImmediate;
      }
    }, 1000);
  };
  return execImmediate;
}();

Answer 3

The threads module should be just what you need:

https://github.com/robtweed/threads