NodeJS request multiple api endpoints

Question

Ok so I am trying to make two or more requests to API endpoints using the request module. I am rendering a HTML file and passing the returned JSON to a handlebars template using the below code:

res.render('list.html', {
  title: 'List',
  data: returnedJSON
}

I can then iterate over this JSON within my handlebars template fairly easily.

The problem I have, is that I am now needing to use multiple data sources where a category list will be built from a categories JSON response and a Staff list from a staff JSON response. I would like a simple solution where I can do this, but expand it to use any number of data sources.

Below is a full code snippet of what I have with one data source:

request({
    url: 'https://api.com/categories',
    headers: {
        'Bearer': 'sampleapitoken'
    }
}, function(error, response, body) {
    if(error || response.statusCode !== 200) {
        // handle error
    } else {
        var json = JSON.parse(body);
        res.render('list.html', {
            title: 'Listing',
            data: json
        });
    }
});

This works great for one endpoint, but as mentioned before I now need to use multiple requests and have multiple data sources for example:

request({
    url: ['https://api.com/categories','https://api.com/staff'],
    headers: {
        'Bearer': 'sampleapitoken'
    }
}, function(error, response, body1, body2) {
    if(error || response.statusCode !== 200) {
        // handle error
    } else {
        var json1 = JSON.parse(body1);
        var json2 = JSON.parse(body2);
        res.render('list.html', {
            title: 'Listing',
            staff: json1,
            categories: json2
        });
    }
});

I appreciate the above doesn't work like that, but I hope this can help convey what I am trying to achieve.

Thanks in advance :)

Answer 1

You can use the async library to map your request objects and pass them to an actual request and return all results in one callback.

var async = require("async");
var request = require("request");

// create request objects
var requests = [{
  url: 'https://api.com/categories',
  headers: {
    'Bearer': 'sampleapitoken'
  }
}, {
  url: 'https://api.com/staff',
  headers: {
    'Bearer': 'sampleapitoken'
  }
}];

async.map(requests, function(obj, callback) {
  // iterator function
  request(obj, function(error, response, body) {
    if (!error && response.statusCode == 200) {
      // transform data here or pass it on
      var body = JSON.parse(body);
      callback(null, body);
    } else {
      callback(error || response.statusCode);
    }
  });
}, function(err, results) {
  // all requests have been made
  if (err) {
    // handle your error
  } else {
    console.log(results);
    for (var i = 0; i < results.length; i++) {
      // request body is results[i]
    }
  }
});

---

However a simpler way would to leverage promises, this can be done with bluebird and promisifying the request lib, or use the already promisified request lib request-promise. You'll still want to include a promise/A+ lib to map the results asynchronously.

var Promise = require("bluebird");
var request = require('request-promise');

// create request objects
var requests = [{
  url: 'https://api.com/categories',
  headers: {
    'Bearer': 'sampleapitoken'
  }
}, {
  url: 'https://api.com/staff',
  headers: {
    'Bearer': 'sampleapitoken'
  }
}];

Promise.map(requests, function(obj) {
  return request(obj).then(function(body) {
    return JSON.parse(body);
  });
}).then(function(results) {
  console.log(results);
  for (var i = 0; i < results.length; i++) {
    // access the result's body via results[i]
  }
}, function(err) {
  // handle all your errors here
});

---

It's important to note that all latest versions of node and browsers support Promises out of the box and this can be implemented without external libraries.

Answer 2

Seems like promises could help.

The easiest would probably be to create a new request method that returns a promise (or promisifying with Bluebird etc), then wait for all promises to finish, and handle the data

function doReq(url, what) {
    return new Promise(function(resolve, reject) {
        request({
            url: url,
            headers: {
                'Bearer': 'sampleapitoken'
            }
        }, function(error, response) {
            if(error || response.statusCode !== 200) {
                reject(error);
            } else {
                var data = {};
                (Array.isArray(what) ? what : [what]).forEach(function(item, index) {
                    data[item] = JSON.parse(arguments[index + 2]);
                });
                resolve( data );
            }
        });
    });
}

Promise.all([
    doReq('https://api.com/categories', 'data'), 
    doReq(['https://api.com/categories','https://api.com/staff'], ['staff', 'categories'])
]).then(function() {
    var obj = {title : 'Listing'};
    [].slice.call(arguments).forEach(function(arg) {
        Object.keys(arg).forEach(function(key) {
            obj[key] = arg[key];
        });
    });
    res.render('list.html', obj);
});