I am using the Stackoverflow API to collect the tags in Stackoverflow. I am using NodeJS and am facing a typical problem. My rate limit gets exceeded after 230-250 requests.AS far as I have read it allows <30req/sec . So it should work fine according to my program, but instead I get an exception after 200+ cycles.MY nodejs code is as follows:-
var request=require('superagent');
var request=require('mongoose');
mongoose.connect('mongodb://shivkumarganesh:[email protected]:10090/lklklklkkkkl_gshiv_sk');
var db = mongoose.connection;
db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', function callback () {
console.log('Connected');
});
//Schema for Tags
var tagSchema = mongoose.Schema({
name:String,
count:Number,
is_required:Boolean
, is_moderator_only:Boolean,
has_synonyms:Boolean});
var Tags = mongoose.model('Tags', tagSchema);
var saveSchema = function(Model){
Model.save();
}
///usr/local/lib/node_modules/superagent
var i=1;
var req=function(){
request.get('http://api.stackexchange.com/2.1/tags?page='+i+'&pagesize=100&fromdate=1199145600&todate=1377648000&order=asc&sort=popular&site=stackoverflow')
.set('X-API-Key', 'foobar')
.set('Accept', 'application/json')
.end(function(res){
if (res.ok && i<346) {
console.log('yay got ' + res.body);
var jsonResponse = res.body;
console.log(jsonResponse.items.length);
for(var j=0;j<jsonResponse.items.length;j++)
{
var model = new Tags(jsonResponse.items[j]);
saveSchema(model);
}
i++;
console.log("Number of Iteration ---- > "+i);
setTimeout(function() {
req();
}, 3000);
} else {
console.log('Oh no! error ' + res.text);
console.log('Processing Over'+i)
}
});
}
req();
Please help me to figure this out.If possible please rectify the program. Note: I amusing MongoHQ to store data.
You should read this informative link: https://api.stackexchange.com/docs/throttle
Specifically:
So check out the
backoff
field returned, and wait for that much before doing another request.