1 Reply Latest reply on Oct 18, 2018 10:55 AM by patrick.byrne.0

    Looping GET requests in getData function for Web Data Connector

    Jonathan Yi

      Hello,

       

      I'm having difficulty with the getData() function while trying to use a Tableau Web Data Connector.

       

      Essentially, I'm trying to send a call to GET a list of db ids, then loop the response list to send individual GET requests (using the ith id in the list as a parameter for the request). I started off with the WDC earthquake example and modified it, here is my first (simplified) getData implementation

       

      (function () {
       var docIds = [];
       var tableData = [];
      
       var myConnector = tableau.makeConnector();
      
       myConnector.init = function(initCallback) {
           initCallback();
           tableau.submit();
       };
      
       myConnector.getSchema = function (schemaCallback) {
           var cols = [
            {
               id: "id",
               dataType: tableau.dataTypeEnum.string
            },
            //{}, ... define other schema columns here, actual schema omitted for brevity
           ];
           var tableSchema = {
                id: "earthquakeFeed",
                alias: "Earthquakes with magnitude greater than 4.5 in the last seven days",
                columns: cols
           };
           schemaCallback([tableSchema]);
       };
      
       myConnector.getData = function (table, doneCallback) {
           $.get('http://localhost:8888/_all_dbs', function(result, status, xhrObj){
                docIds = result;
                // ... do some filtering on docIds
           }).done(function(result2, status2, jqXhr){
                for(var i=0; i<docIds.length; i++){     // iterate doc ids
                     $.get('http://localhost:8888/'+docIds[i]+'/_all_docs?include_docs=true', function(result3, status3, xhrObj3){
                          var data = result3;
                          // ... parse data according to schema
                          tableData.push(data);
                          if(i+1 == docIds.length){     // last index
                               table.appendRows(tableData);
                               doneCallback();
                          }
                     });
                }
           });
       });
      
       tableau.registerConnector(myConnector);
      })();
      

       

      The issue I was seeing with this implementation seems to be that there were too many requests sent at one shot. The first GET call returns a list with about 10,000 ids. Also I believe the tableData object was too large for memory.

       

      Next I tried a few different approaches, (adding a setTimeout and sending the requests in smaller "chunks") which did not work.

       

      What I ended up with, I adjusted the getData function to recursively get each response (and instead of pushing each to an array that's used as input for table.appendRows, I call table.appendRows on each document row).

       

      When I try the following code with a hard set limit on number of ids, the WDC works (tested with the Simulator tutorial and also in Tableau Desktop). If I hard set the loop to 1000 iterations, getting the datasource in Tableau Desktop is successful, but when I try to get all the documents from all the ids in the list (length of the array is ~10K), Tableau desktop hangs while trying to get the data source.

       

      (function () {
           var docIds = [];
           var getAllDocsOfNextId;
      
           var myConnector = tableau.makeConnector();
           myConnector.init = function(initCallback) {
                initCallback();
                tableau.submit();
           };
      
           myConnector.getSchema = function (schemaCallback) {
                var cols = [
                   {
                    id: "id",
                     dataType: tableau.dataTypeEnum.string
                   },
                   // ... define other schema columns here (omitted for brevity)
                ];
                var tableSchema = {
                     id: "earthquakeFeed",
                     alias: "Earthquakes with magnitude greater than 4.5 in the last seven days",
                     columns: cols
                };
                schemaCallback([tableSchema]);
           };
      
           myConnector.getData = function (table, doneCallback) {
                $.get('http://localhost:8888/_all_dbs', function(result1, status, xhrObj){
                     docIds = result1;
                     // ... do some filtering on docIds array to remove some elements
                }).done(function(result2, status2, jqXhr){
                     getAllDocsOfNextId = function(index){
                          // ajax get all docs of id
                          $.get('http://localhost:8888/'+docIds[index]+'/_all_docs?include_docs=true', function(result3, status3, xhrObj3){
                               var rows = result3.rows;
                               for(var i=0; i<rows.length; i++){ // iterate projects in solution
                                    var doc = JSON.parse(JSON.stringify(rows[i].doc));
                                    table.appendRows([{
                                         "docId":doc._id,
                                         //... assign other schema fields from doc values
                                    }]);
                               }
                               //if(index < docIds.length){ // this doesn't work in Tableau
                               if(index < 1000){
                                    index++;
                                    getAllDocsOfNextId(index); // get next db's documents
                               }
                               else{ // done iterating ids
                                    doneCallback();
                               }
                          })
                     }
                     getAllDocsOfNextId(0); // get first db's documents, recursively iterate
                });
           };
      
           tableau.registerConnector(myConnector);
      })();
      

       

      If anyone can show me the correct approach on how to send many requests and accumulate a very large object for the table.appendRows part in the getData function for Web Data Connector, I would be very grateful.

       

      Thanks in advance,

      Jon