How to do repeated requests until one succeeds without blocking in node?

Question

I have a function that takes a parameter and a callback. It\'s supposed to do a request to a remote API and get some info based on the parameter. When it gets the info, it needs

Answer 1

There is no need to re-invent the wheel... you can use a popular async utility library, 'retry' method in this case.

// try calling apiMethod 3 times
async.retry(3, apiMethod, function(err, result) {
    // do something with the result
});

// try calling apiMethod 3 times, waiting 200 ms between each retry
async.retry({times: 3, interval: 200}, apiMethod, function(err, result) {
    // do something with the result
});

async GitHub page

async.retry docs

Answer 2

Is this what you are trying to do?

var history = {};

function sendRequest(options, callback) {
    var req = https.request(options, function (res) {
        var acc = "";
        res.on("data", function (msg) {
            acc += msg.toString("utf-8");
        });
        res.on("end", function () {
            history = JSON.parse(acc);
            if (history.success) {
                callback(history);
            }
            else {
                sendRequest(options, callback);
            }
        });
    });
    req.end();
}

sendRequest(options, callback);

Answer 3

I found Dmitry's answer using the async utility library very useful and the best answer.

This answer expands his example to a working version that defines the apiMethod function and passes it a parameter. I was going to add the code as a comment but a separate answer is clearer.

const async = require('async');

const apiMethod = function(uri, callback) {
  try {
    // Call your api here (or whatever thing you want to do) and assign to result.
    const result = ...
    callback(null, result);
  } catch (err) {
    callback(err);
  }
};

const uri = 'http://www.test.com/api';

async.retry(
    { times: 5, interval: 200 },
    function (callback) { return apiMethod(uri, callback) },
    function(err, result) {
      if (err) {
        throw err; // Error still thrown after retrying N times, so rethrow.
      }
  });

Retry documentation: https://caolan.github.io/async/docs.html#retry

Note, an alternative to calling apiMethod(uri, callback) in the task is to use async.apply:

async.retry(
        {times: 5, interval: 200},
        async.apply(task, dir),
        function(err, result) {
          if (err) {
            throw err; // Error still thrown after retrying N times, so rethrow.
          }
      });

I hope this provides a good copy/paste boiler plate solution for someone.

Answer 4

I've solved this problem using the retry module.

Example:

var retry = require('retry');

// configuration
var operation = retry.operation({
  retries: 2,           // try 1 time and retry 2 times if needed, total = 3
  minTimeout: 1 * 1000, // the number of milliseconds before starting the first retry
  maxTimeout: 3 * 1000  // the maximum number of milliseconds between two retries
});

// your unreliable task
var task = function(input, callback) {

  Math.random() > 0.5
    ? callback(null, 'ok')   // success
    : callback(new Error()); // error
}

// define a function that wraps our unreliable task into a fault tolerant task
function faultTolerantTask(input, callback) {

  operation.attempt(function(currentAttempt) {

    task(input, function(err, result) {

      console.log('Current attempt: ' + currentAttempt);

      if (operation.retry(err)) {  // retry if needed
          return;
      }

      callback(err ? operation.mainError() : null, result);
    });
  });
}

// test
faultTolerantTask('some input', function(err, result) {
  console.log(err, result);
});

Answer 5

Definitely not the way to go - while(!done); will go into a hard loop and take up all of your cpu.

Instead you could do something like this (untested and you may want to implement a back-off of some sort):

function tryUntilSuccess(options, callback) {
    var req = https.request(options, function(res) {
        var acc = "";
        res.on("data", function(msg) {
            acc += msg.toString("utf-8");
        });
        res.on("end", function() {
            var history = JSON.parse(acc);  //<== Protect this if you may not get JSON back
            if (history.success) {
                callback(null, history);
            } else {
                tryUntilSuccess(options, callback);
            }
        });
    });
    req.end();

    req.on('error', function(e) {
        // Decide what to do here
        // if error is recoverable
        //     tryUntilSuccess(options, callback);
        // else
        //     callback(e);
    });
}

// Use the standard callback pattern of err in first param, success in second
tryUntilSuccess(options, function(err, resp) {
    // Your code here...
});

Answer 6

You could try something along the following lines. I'm writing a general idea, you should replace trySomething with your HTTP request.

function keepTrying(onSuccess) {
  function trySomething(onSuccess, onError) {
    if (Date.now() % 7 === 0) {
      process.nextTick(onSuccess);
    } else {
      process.nextTick(onError);
    }
  }
  trySomething(onSuccess, function () {
    console.log('Failed, retrying...');
    keepTrying(onSuccess);
  });
}

keepTrying(function () {
  console.log('Succeeded!');
});

I hope this helps.