javascript - Batch requests in Node.js -

my program communicating web service accepts ~10 requests per second. time time, program sends 100+ concurrent requests web service, causing program crash.

how limit concurrent requests in node.js 5 per second? im using request library.

 // if event , sender     if(data.sender[0].events && data.sender[0].events.length > 0) {           // find events         for(var = 0; < data.sender[0].events.length; i++) {              // if type "added"             if(data.sender[0].events[i].type == "added") {                 switch (data.sender[0].events[i].link.rel) {                     case "contact" :                         batch("added", data.sender[0].events[i].link.href);                         //_initcontacts(data.sender[0].events[i].link.href);                         break;                 }              // if type "updated"             } else if(data.sender[0].events[i].type == "updated") {                  switch (data.sender[0].events[i].link.rel){                                          case "contactpresence" :                         batch("updated", data.sender[0].events[i].link.href);                         //_getcontactpresence(data.sender[0].events[i].link.href);                         break;                     case "contactnote" :                         batch("updated", data.sender[0].events[i].link.href);                         // _getcontactnote(data.sender[0].events[i].link.href);                         break;                     case "contactlocation" :                         batch("updated", data.sender[0].events[i].link.href);                         // _getcontactlocation(data.sender[0].events[i].link.href);                         break;                     case "presencesubscription" :                         batch("updated", data.sender[0].events[i].link.href);                         // _extendpresencesubscription(data.sender[0].events[i].link.href);                         break;                 }             }         }; 

and homegrown batch method:

var updated = []; var added = [];  var batch = function(type, url){     console.log("batch called");       if (type === "added"){         console.log("added batched");         added.push(url);         if (added.length > 5) {             settimeout(added.foreach(function(req){                 _initcontacts(req);             }), 2000);             added = [];         }     }      else if (type === "updated"){         console.log("updated batched");         updated.push(url);         console.log("updated length : ", updated.length);         if (updated.length > 5){             console.log("over 5 updated events");             updated.foreach(function(req){                 settimeout(_getcontactlocation(req), 2000);             });             updated = [];         }     }        }; 

and example of actual request:

var _getcontactlocation = function(url){     r.get(baseurl + url,      { "strictssl" : false, "headers" : { "authorization" : "bearer " + accesstoken }},          function(err, res, body){             if(err)                 console.log(err);             else {                 var data = json.parse(body);                 self.emit("data.contact", data);             }         }     ); }; 

using async library, maplimit function want. can't provide example specific use case did not provide code.

from readme:

---

maplimit(arr, limit, iterator, callback)

the same map no more "limit" iterators simultaneously running @ time.

note items not processed in batches, there no guarantee first "limit" iterator functions complete before others started.

arguments

• arr - array iterate over.
• limit - maximum number of iterators run @ time.
• iterator(item, callback) - function apply each item in array. iterator passed callback(err, transformed) must called once has completed error (which can null) , transformed item.
• callback(err, results) - callback called after iterator functions have finished, or error has occurred. results array of transformed items original array.

example

async.maplimit(['file1','file2','file3'], 1, fs.stat, function(err, results){ // results array of stats each file });

---

edit: provided code, see use bit different assumed. async library more useful when know tasks run front. don't know of library off hand solve you. above note still relevant people searching topic i'll leave in.

sorry, don't have time restructure code, (un-tested) example of function makes asynchronous request while self-throttling 5 requests per second. highly recommend working off of come more general solution fits code base.

var throttledrequest = (function () {     var queue = [], running = 0;      function sendpossiblerequests() {         var url;         while (queue.length > 0 && running < 5) {             url = queue.shift();             running++;             r.get(url, { /* options here*/ }, function (err, res, body) {                 running--;                 sendpossiblerequests();                  if(err)                     console.log(err);                 else {                     var data = json.parse(body);                     self.emit("data.contact", data);                 }             });         }     }      return function (url) {         queue.push(url);         sendpossiblerequests();     }; })(); 

basically, keep queue of data asynchronously processed (such urls requested) , after each callback (from request) try launch off many remaining requests possible.