d3.js queue.js and node.js

Question

I'm trying to load data in nodejs before passing it to expressjs to render a d3 chart in the browser.

I understand that I can load data this way from this - https://github.com/mbostock/queue

I have an expressjs route configurd like this -

    var d3 = require('d3')
            ,queue = require("queue-async")
            ;

        router.get('/', handler1);

        function handler1(req, res) {

    queue()
       .defer(d3.json, 'https://data.medicare.gov/resource/a8s4-5eya.json?$limit=50000&$$app_token=igGS35vW9GvDMmJUnmHju2MEH&$select=org_pac_id%20as%20source, org_pac_id%20as%20target,org_lgl_nm%20as%20description')
       .defer(d3.json, 'https://data.medicare.gov/resource/a8s4-5eya.json?$limit=50000&$$app_token=igGS35vW9GvDMmJUnmHju2MEH&$select=org_pac_id%20as%20source, org_pac_id||pri_spec%20as%20target, pri_spec%20as%20description')
       .defer(d3.json, 'https://data.medicare.gov/resource/a8s4-5eya.json?$limit=50000&$$app_token=igGS35vW9GvDMmJUnmHju2MEH&$select=org_pac_id||pri_spec%20as%20tsource, pri_spec%20as%20target, frst_nm%20as%20description')
       .await(go);

    function go(error, data,d2,d3){

     data.concat(d2); data.concat(d3);

     console.log(data);

     res.render('index', { title: 'Group Practices', data });

    }  
  }
module.exports = router;

But am getting a browser error,

XMLHttpRequest is not defined

ReferenceError: XMLHttpRequest is not defined
    at d3_xhr (/Users/Admin/Public/GroupPractice/node_modules/d3/d3.js:1934:114)
    at d3.json (/Users/Admin/Public/GroupPractice/node_modules/d3/d3.js:9533:12)
    at pop (/Users/Admin/Public/GroupPractice/node_modules/queue-async/queue.js:24:14)
    at Object.q.defer (/Users/Admin/Public/GroupPractice/node_modules/queue-async/queue.js:55:11)
    at handler1 (/Users/Admin/Public/GroupPractice/routes/index.js:18:5)
    at Layer.handle [as handle_request] (/Users/Admin/Public/GroupPractice/node_modules/express/lib/router/layer.js:95:5)
    at next (/Users/Admin/Public/GroupPractice/node_modules/express/lib/router/route.js:131:13)
    at Route.dispatch (/Users/Admin/Public/GroupPractice/node_modules/express/lib/router/route.js:112:3)
    at Layer.handle [as handle_request] (/Users/Admin/Public/GroupPractice/node_modules/express/lib/router/layer.js:95:5)
    at /Users/Admin/Public/GroupPractice/node_modules/express/lib/router/index.js:277:22

How do I use d3 and queue to pre-load this RESTful data?

Answer 1

The d3 library is a client-side one. This means that it must be used within a browser not within a server-side application.

If you want to get data and concat them within your Express application, you could use async and request as described below:

var request = require('request');
var async = require('async');

function createRequestTask(url) {
  return function(callback) {
    request({
      url: url,
      json: true        
    }, function (err, response, body) {
      callback(err, body);
    });
  };
}

async.parallel([
  // URL #1
  createRequestTask('https://data.medicare.gov/resource/a8s4-5eya.json?$limit=50000&$$app_token=igGS35vW9GvDMmJUnmHju2MEH&$select=org_pac_id%20as%20source, org_pac_id%20as%20target,org_lgl_nm%20as%20description'),
  // URL #2
  createRequestTask('https://data.medicare.gov/resource/a8s4-5eya.json?$limit=50000&$$app_token=igGS35vW9GvDMmJUnmHju2MEH&$select=org_pac_id%20as%20source, org_pac_id%20as%20target,org_lgl_nm%20as%20description'),
  (...)
],
function(err, results) {
  // Result contains an array with each request result
});

The requests will be executed in parallel and when all requests are finished, the final callback is called. The results variable will contain an array with all response contents.

You can then leverage this array to build the object you want the Express route returns.

Here is the update of your code:

router.get('/', handler1);

function handler1(req, res) {
  async.parallel([
    // URL #1
    createRequestTask('https://data.medicare.gov/...'),
    // URL #2
    createRequestTask('https://data.medicare.gov/...'),
    (...)
  ],
  function(err, results) {
    res.render('index', { title: 'Group Practices', results });
  });  
}

module.exports = router;

Hope it helps you, Thierry

Answer 2

I got around this issue by using xmlhttprequest npm package and then inside my d3.js file in my node_modules folder I added this at line 1934 where the d3_xhr function is defined.

var XMLHttpRequest = require('xmlhttprequest').XMLHttpRequest;

That way my d3 can always make xhr requests while I'm working in node. I hope more people find this because it's a super easy fix that not many people seem to think about doing to overcome the lack of built-in xhr support in node.