A few performance improvements for events with a large number of fields

src/app/components/kbn.js

@@ -13,15 +13,10 @@ function($, _, moment) {
     return field_array.sort();
   };
 
-  kbn.get_all_fields = function(data) {
+  kbn.get_all_fields = function(data,flat) {
-    var _d = data;
+    return _.uniq(_.without(_.reduce(data,function(memo,hit) {
-    var fields = [];
+      return flat ? memo.concat(_.keys(kbn.flatten_json(hit._source))) : memo.concat(_.keys(hit._source));
-    _.each(_d,function(hit) {
+    },[]),'$$hashkey'));
-      fields = _.uniq(fields.concat(_.keys(kbn.flatten_json(hit._source))));
-    });
-    // Remove stupid angular key
-    fields = _.without(fields,'$$hashKey');
-    return fields;
   };
 
   kbn.has_field = function(obj,field) {

src/app/panels/table/module.html

@@ -19,11 +19,6 @@
 
 
         <ul class="unstyled" style="{{panel.overflow}}:{{panel.height || row.height}};overflow-y:auto;overflow-x:hidden;">
-          <li ng-style="panel.style" ng-repeat="field in metaFields|filter:fieldFilter|orderBy:identity">
-            <i class="pointer" ng-class="{'icon-check': _.contains(panel.fields,field),'icon-check-empty': !_.contains(panel.fields,field)}" ng-click="toggle_field(field)"></i>
-            <a class="pointer" data-unique="1" bs-popover="'app/panels/table/micropanel.html'" data-placement="rightTop" ng-click="toggle_micropanel(field,true)" ng-class="{label: _.contains(panel.fields,field)}">{{field}}</a>
-          </li>
-
           <li ng-style="panel.style" ng-repeat="field in fields.list|filter:fieldFilter|orderBy:identity" ng-show="panel.all_fields">
             <i class="pointer" ng-class="{'icon-check': _.contains(panel.fields,field),'icon-check-empty': !_.contains(panel.fields,field)}" ng-click="toggle_field(field)"></i>
             <a class="pointer" data-unique="1" bs-popover="'app/panels/table/micropanel.html'" data-placement="rightTop" ng-click="toggle_micropanel(field,true)" ng-class="{label: _.contains(panel.fields,field)}">{{field}}</a>
@@ -68,7 +63,7 @@
         </thead>
         <tbody ng-repeat="event in data| slice:panel.offset:panel.offset+panel.size" ng-class-odd="'odd'">
           <tr ng-click="toggle_details(event)" class="pointer">
-            <td ng-show="panel.fields.length<1">{{event.kibana._source|stringify|tableTruncate:panel.trimFactor:1}}</td>
+            <td ng-show="panel.fields.length<1">{{event._source|stringify|tableTruncate:panel.trimFactor:1}}</td>
             <td ng-show="panel.fields.length>0" ng-repeat="field in panel.fields" ng-bind-html-unsafe="(event.kibana.highlight[field]||event.kibana._source[field]) |tableHighlight | tableTruncate:panel.trimFactor:panel.fields.length"></td>
           </tr>
           <tr ng-show="event.kibana.details">

src/app/panels/table/module.js

@@ -87,7 +87,6 @@ function (angular, app, _, kbn, moment) {
 
     $scope.init = function () {
       $scope.Math = Math;
-      $scope.metaFields = [];
       $scope.identity = angular.identity;
       $scope.$on('refresh',function(){$scope.get_data();});
 
@@ -258,6 +257,7 @@ function (angular, app, _, kbn, moment) {
         if(_segment === 0) {
           $scope.hits = 0;
           $scope.data = [];
+          $scope.current_fields = [];
           query_id = $scope.query_id = new Date().getTime();
         }
 
@@ -269,21 +269,26 @@ function (angular, app, _, kbn, moment) {
 
         // Check that we're still on the same query, if not stop
         if($scope.query_id === query_id) {
-          $scope.data= $scope.data.concat(_.map(results.hits.hits, function(hit) {
+
+          // This is exceptionally expensive, especially on events with a large number of fields
+          $scope.data = $scope.data.concat(_.map(results.hits.hits, function(hit) {
             var
               _h = _.clone(hit),
               _p = _.omit(hit,'_source','sort','_score');
 
-            $scope.metaFields = _.union(_.keys(_p),$scope.metaFields);
-
             // _source is kind of a lie here, never display it, only select values from it
             _h.kibana = {
               _source : _.extend(kbn.flatten_json(hit._source),_p),
               highlight : kbn.flatten_json(hit.highlight||{})
             };
+
+            // Kind of cheating with the _.map here, but this is faster than kbn.get_all_fields
+            $scope.current_fields = $scope.current_fields.concat(_.keys(_h.kibana._source));
+
             return _h;
           }));
 
+          $scope.current_fields = _.uniq($scope.current_fields);
           $scope.hits += results.hits.total;
 
           // Sort the data
@@ -303,9 +308,6 @@ function (angular, app, _, kbn, moment) {
           // Keep only what we need for the set
           $scope.data = $scope.data.slice(0,$scope.panel.size * $scope.panel.pages);
 
-          // Populate current_fields list
-          $scope.current_fields = kbn.get_all_fields($scope.data);
-
         } else {
           return;
         }

src/app/services/fields.js

@@ -21,6 +21,7 @@ function (angular, _, config) {
         var indices = _.difference(n,_.keys(self.mapping));
         // Only get the mapping if there are new indices
         if(indices.length > 0) {
+          console.log('getting mapping');
           self.map(indices).then(function(result) {
             self.mapping = _.extend(self.mapping,result);
             self.list = mapFields(self.mapping);