Iterate over paged data sets

kpdecker · kpdecker · commit ffc4af1736fa · 2013-08-27T17:05:27.000-05:00
Some APIs do not honor the per_page API (at least for large values) and
require that we actually walk the links headers to pull all data.
diff --git a/github.js b/github.js
@@ -35,7 +35,7 @@
 
     function _request(method, path, data, cb, raw) {
       function getURL() {
-        var url = API_URL + path;
+        var url = path.indexOf('//') >= 0 ? path : API_URL + path;
         return url + ((/\?/).test(url) ? "&" : "?") + (new Date()).getTime();
       }
 
@@ -46,9 +46,9 @@
       xhr.onreadystatechange = function () {
         if (this.readyState == 4) {
           if (this.status >= 200 && this.status < 300 || this.status === 304) {
-            cb(null, raw ? this.responseText : this.responseText ? JSON.parse(this.responseText) : true);
+            cb(null, raw ? this.responseText : this.responseText ? JSON.parse(this.responseText) : true, this);
           } else {
-            cb({request: this, error: this.status});
+            cb({path: path, request: this, error: this.status});
           }
         }
       };
@@ -66,14 +66,42 @@
       data ? xhr.send(JSON.stringify(data)) : xhr.send();
     }
 
+    function _requestAllPages(path, cb) {
+      var results = [];
+      (function iterate() {
+        _request("GET", path, null, function(err, res, xhr) {
+          if (err) {
+            return cb(err);
+          }
+
+          results.push.apply(results, res);
+
+          var links = (xhr.getResponseHeader('link') || '').split(/\s*,\s*/g),
+              next = _.find(links, function(link) { return /rel="next"/.test(link); });
+
+          if (next) {
+            next = (/<(.*)>/.exec(next) || [])[1];
+          }
+
+          if (!next) {
+            cb(err, results);
+          } else {
+            path = next;
+            iterate();
+          }
+        });
+      })();
+    }
+
 
 
     // User API
     // =======
 
     Github.User = function() {
       this.repos = function(cb) {
-        _request("GET", "/user/repos?type=all&per_page=1000&sort=updated", null, function(err, res) {
+        // Github does not always honor the 1000 limit so we want to iterate over the data set.
+        _requestAllPages("/user/repos?type=all&per_page=1000&sort=updated", function(err, res) {
           cb(err, res);
         });
       };
@@ -111,7 +139,8 @@
       // -------
 
       this.userRepos = function(username, cb) {
-        _request("GET", "/users/"+username+"/repos?type=all&per_page=1000&sort=updated", null, function(err, res) {
+        // Github does not always honor the 1000 limit so we want to iterate over the data set.
+        _requestAllPages("/users/"+username+"/repos?type=all&per_page=1000&sort=updated", function(err, res) {
           cb(err, res);
         });
       };
@@ -129,7 +158,8 @@
       // -------
 
       this.orgRepos = function(orgname, cb) {
-        _request("GET", "/orgs/"+orgname+"/repos?type=all&per_page=1000&sort=updated&direction=desc", null, function(err, res) {
+        // Github does not always honor the 1000 limit so we want to iterate over the data set.
+        _requestAllPages("/orgs/"+orgname+"/repos?type=all&&page_num=1000&sort=updated&direction=desc", function(err, res) {
           cb(err, res);
         });
       };
