From 170dfe615883d869d1da7e581dfdbc9ce191afd6 Mon Sep 17 00:00:00 2001 From: Dmitry Parfenchik Date: Mon, 5 Jun 2017 22:42:41 +0200 Subject: [PATCH 1/6] History UI: Improving performance by detaching table DOM before processing Currently all the DOM manipulations are handled in a loop after Mustache template is parsed. This causes severe performance issues especially within loops iteration over thousands of (attempt/application) records and causing all kinds of unnecessary browser work: reflow, repaint, etc. This could be easily fixed by preparing a DOM node beforehand and doing all manipulations within the loops on detached node, reattaching it to the document only after the work is done. The most costly operation in this case was setting innerHTML for `duration` column within a loop, which is extremely unperformant: https://jsperf.com/jquery-append-vs-html-list-performance/24 While duration parsing could be done before mustache-template processing without any additional DOM alteratoins. --- .../spark/ui/static/historypage-template.html | 2 +- .../org/apache/spark/ui/static/historypage.js | 17 ++++++----------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html index c2afa993b2f20..6833635033533 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html @@ -74,7 +74,7 @@ {{attemptId}} {{startTime}} {{endTime}} - {{duration}} + {{duration}} {{sparkUser}} {{lastUpdated}} Download diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index a4430347a0b2a..76bc1bce61f77 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -116,7 +116,7 @@ $(document).ready(function() { attempt["lastUpdated"] = formatDate(attempt["lastUpdated"]); attempt["log"] = uiRoot + "/api/v1/applications/" + id + "/" + (attempt.hasOwnProperty("attemptId") ? attempt["attemptId"] + "/" : "") + "logs"; - + attempt["duration"] = formatDuration(attempt["duration"]); var app_clone = {"id" : id, "name" : name, "num" : num, "attempts" : [attempt]}; array.push(app_clone); } @@ -128,7 +128,7 @@ $(document).ready(function() { } $.get("static/historypage-template.html", function(template) { - historySummary.append(Mustache.render($(template).filter("#history-summary-template").html(),data)); + var apps = $(Mustache.render($(template).filter("#history-summary-template").html(),data)); var selector = "#history-summary-table"; var conf = { "columns": [ @@ -158,31 +158,26 @@ $(document).ready(function() { if (hasMultipleAttempts) { jQuery.extend(conf, rowGroupConf); - var rowGroupCells = document.getElementsByClassName("rowGroupColumn"); + var rowGroupCells = apps.find(".rowGroupColumn"); for (i = 0; i < rowGroupCells.length; i++) { rowGroupCells[i].style='background-color: #ffffff'; } } if (!hasMultipleAttempts) { - var attemptIDCells = document.getElementsByClassName("attemptIDSpan"); + var attemptIDCells = apps.find(".attemptIDSpan"); for (i = 0; i < attemptIDCells.length; i++) { attemptIDCells[i].style.display='none'; } } if (requestedIncomplete) { - var completedCells = document.getElementsByClassName("completedColumn"); + var completedCells = apps.find(".completedColumn"); for (i = 0; i < completedCells.length; i++) { completedCells[i].style.display='none'; } } - - var durationCells = document.getElementsByClassName("durationClass"); - for (i = 0; i < durationCells.length; i++) { - var timeInMilliseconds = parseInt(durationCells[i].title); - durationCells[i].innerHTML = formatDuration(timeInMilliseconds); - } + historySummary.append(apps); if ($(selector.concat(" tr")).length < 20) { $.extend(conf, {paging: false}); From 07c6a3f57dfcc659d41c59bb394dbc3f8fa989e3 Mon Sep 17 00:00:00 2001 From: Dmitry Parfenchik Date: Sun, 30 Jul 2017 16:52:24 +0200 Subject: [PATCH 2/6] Performance optimization for pagination check Check whether to display pagination or not on large data sets (10-20k rows) was taking up to 50ms because it was iterating over all rows. This could be easily done by testing length of array before passing it to mustache. --- .../resources/org/apache/spark/ui/static/historypage.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index 76bc1bce61f77..0019ce656f545 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -121,6 +121,9 @@ $(document).ready(function() { array.push(app_clone); } } + if(array.length < 20) { + $.fn.dataTable.defaults.paging = false; + } var data = { "uiroot": uiRoot, @@ -178,11 +181,6 @@ $(document).ready(function() { } } historySummary.append(apps); - - if ($(selector.concat(" tr")).length < 20) { - $.extend(conf, {paging: false}); - } - $(selector).DataTable(conf); $('#hisotry-summary [data-toggle="tooltip"]').tooltip(); }); From 14da1621e17d0301a837a24a3307db8f43a0a102 Mon Sep 17 00:00:00 2001 From: Dmitry Parfenchik Date: Sun, 30 Jul 2017 17:23:37 +0200 Subject: [PATCH 3/6] Performance improvement: removing unnecessary DOM processing Logic related to `hasMultipleAttempts` flag: - Hiding attmptId column (if `hasMultipleAttempts = false`) - Seting white background color for first 2 columns (if `hasMultipleAttempts = true`) was updating DOM after mustache template processing, which was causing 2 unnecessary iterations over full data set (first through jquery selector, than through for-loop). Refactoring it inside mustache template helps saving 80-90ms on large data sets (10k+ rows) --- .../spark/ui/static/historypage-template.html | 12 ++- .../org/apache/spark/ui/static/historypage.js | 77 ++++++++++--------- 2 files changed, 48 insertions(+), 41 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html index 6833635033533..2ef18069e8de1 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html @@ -29,11 +29,13 @@ App Name - + {{#hasMultipleAttempts}} + Attempt ID + {{/hasMultipleAttempts}} Started @@ -68,10 +70,12 @@ {{#applications}} - {{id}} - {{name}} + {{id}} + {{name}} {{#attempts}} - {{attemptId}} + {{#hasMultipleAttempts}} + {{attemptId}} + {{/hasMultipleAttempts}} {{startTime}} {{endTime}} {{duration}} diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index 0019ce656f545..8feb1de0400dd 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -48,6 +48,18 @@ function getParameterByName(name, searchString) { return results === null ? "" : decodeURIComponent(results[1].replace(/\+/g, " ")); } +function removeColumnByName(columns, columnName) { + return columns.filter(function(col) {return col.name != columnName}) +} + +function getColumnIndex(columns, columnName) { + for(var i = 0; i < columns.length; i++) { + if (columns[i].name == columnName) + return i; + } + return -1; +} + jQuery.extend( jQuery.fn.dataTableExt.oSort, { "title-numeric-pre": function ( a ) { var x = a.match(/title="*(-?[0-9\.]+)/)[1]; @@ -127,51 +139,38 @@ $(document).ready(function() { var data = { "uiroot": uiRoot, - "applications": array - } + "applications": array, + "hasMultipleAttempts": hasMultipleAttempts, + } $.get("static/historypage-template.html", function(template) { var apps = $(Mustache.render($(template).filter("#history-summary-template").html(),data)); var selector = "#history-summary-table"; + var attemptIdColumnName = 'attemptId'; + var defaultSortColumn = completedColumnName = 'completed'; + var durationColumnName = 'duration'; var conf = { - "columns": [ - {name: 'first', type: "appid-numeric"}, - {name: 'second'}, - {name: 'third'}, - {name: 'fourth'}, - {name: 'fifth'}, - {name: 'sixth', type: "title-numeric"}, - {name: 'seventh'}, - {name: 'eighth'}, - {name: 'ninth'}, - ], - "columnDefs": [ - {"searchable": false, "targets": [5]} - ], - "autoWidth": false, - "order": [[ 4, "desc" ]] - }; - - var rowGroupConf = { - "rowsGroup": [ - 'first:name', - 'second:name' - ], + "columns": [ + {name: 'appId', type: "appid-numeric"}, + {name: 'appName'}, + {name: attemptIdColumnName}, + {name: 'started'}, + {name: completedColumnName}, + {name: durationColumnName, type: "title-numeric"}, + {name: 'user'}, + {name: 'lastUpdated'}, + {name: 'eventLog'}, + ], + "autoWidth": false, }; if (hasMultipleAttempts) { - jQuery.extend(conf, rowGroupConf); - var rowGroupCells = apps.find(".rowGroupColumn"); - for (i = 0; i < rowGroupCells.length; i++) { - rowGroupCells[i].style='background-color: #ffffff'; - } - } - - if (!hasMultipleAttempts) { - var attemptIDCells = apps.find(".attemptIDSpan"); - for (i = 0; i < attemptIDCells.length; i++) { - attemptIDCells[i].style.display='none'; - } + conf.rowsGroup = [ + 'appId:name', + 'appName:name' + ]; + } else { + conf.columns = removeColumnByName(conf.columns, attemptIdColumnName); } if (requestedIncomplete) { @@ -180,6 +179,10 @@ $(document).ready(function() { completedCells[i].style.display='none'; } } + conf.order = [[ getColumnIndex(conf.columns, defaultSortColumn), "desc" ]]; + conf.columnDefs = [ + {"searchable": false, "targets": [getColumnIndex(conf.columns, durationColumnName)]} + ]; historySummary.append(apps); $(selector).DataTable(conf); $('#hisotry-summary [data-toggle="tooltip"]').tooltip(); From 80c11663c28d183161648de5f7a32fa3ae49cfd8 Mon Sep 17 00:00:00 2001 From: Dmitry Parfenchik Date: Sun, 30 Jul 2017 22:06:32 +0200 Subject: [PATCH 4/6] Performance improvement: further reducing DOM manipulations Refactoring incomplete requests filter behavior due to inefficency in DOM manipulations. We were traversing DOM 2 more times just to hide columns that we could have avoided rendering in mustache. Factoring this logic in mustache template (`showCompletedColumn`) saves 70-80ms on 10k+ rows. --- .../apache/spark/ui/static/historypage-template.html | 8 ++++++-- .../org/apache/spark/ui/static/historypage.js | 11 ++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html index 2ef18069e8de1..fa1c6fc843f5b 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html @@ -41,11 +41,13 @@ Started - + {{#showCompletedColumn}} + Completed + {{/showCompletedColumn}} Duration @@ -77,7 +79,9 @@ {{attemptId}} {{/hasMultipleAttempts}} {{startTime}} - {{endTime}} + {{#showCompletedColumn}} + {{endTime}} + {{/showCompletedColumn}} {{duration}} {{sparkUser}} {{lastUpdated}} diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index 8feb1de0400dd..7b1390550401c 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -141,12 +141,14 @@ $(document).ready(function() { "uiroot": uiRoot, "applications": array, "hasMultipleAttempts": hasMultipleAttempts, + "showCompletedColumn": !requestedIncomplete, } $.get("static/historypage-template.html", function(template) { var apps = $(Mustache.render($(template).filter("#history-summary-template").html(),data)); var selector = "#history-summary-table"; var attemptIdColumnName = 'attemptId'; + var startedColumnName = 'started'; var defaultSortColumn = completedColumnName = 'completed'; var durationColumnName = 'duration'; var conf = { @@ -154,7 +156,7 @@ $(document).ready(function() { {name: 'appId', type: "appid-numeric"}, {name: 'appName'}, {name: attemptIdColumnName}, - {name: 'started'}, + {name: startedColumnName}, {name: completedColumnName}, {name: durationColumnName, type: "title-numeric"}, {name: 'user'}, @@ -173,11 +175,10 @@ $(document).ready(function() { conf.columns = removeColumnByName(conf.columns, attemptIdColumnName); } + var defaultSortColumn = completedColumnName; if (requestedIncomplete) { - var completedCells = apps.find(".completedColumn"); - for (i = 0; i < completedCells.length; i++) { - completedCells[i].style.display='none'; - } + defaultSortColumn = startedColumnName; + conf.columns = removeColumnByName(conf.columns, completedColumnName); } conf.order = [[ getColumnIndex(conf.columns, defaultSortColumn), "desc" ]]; conf.columnDefs = [ From 2f72c98de4c092a29fa3a0eb9bd229d6bada25e5 Mon Sep 17 00:00:00 2001 From: Dmitry Parfenchik Date: Sun, 30 Jul 2017 22:26:59 +0200 Subject: [PATCH 5/6] Performance improvements: detaching DOM before DataTables plugin processing Detaching history table wrapper from document before parsing it with DataTables plugin and reattaching back right after plugin has processed nested DOM. This allows to avoid huge amount of browser repaints and reflows, reducing initial page load time in Chrome from 15s to 4s for 20k+ rows --- .../resources/org/apache/spark/ui/static/historypage.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index 7b1390550401c..adec2f615b8b6 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -145,8 +145,9 @@ $(document).ready(function() { } $.get("static/historypage-template.html", function(template) { + var sibling = historySummary.prev(); + historySummary.detach(); var apps = $(Mustache.render($(template).filter("#history-summary-template").html(),data)); - var selector = "#history-summary-table"; var attemptIdColumnName = 'attemptId'; var startedColumnName = 'started'; var defaultSortColumn = completedColumnName = 'completed'; @@ -185,7 +186,8 @@ $(document).ready(function() { {"searchable": false, "targets": [getColumnIndex(conf.columns, durationColumnName)]} ]; historySummary.append(apps); - $(selector).DataTable(conf); + apps.DataTable(conf); + sibling.after(historySummary); $('#hisotry-summary [data-toggle="tooltip"]').tooltip(); }); }); From e487a4eda4cfbd311f1587ad2852688f94f3b6a9 Mon Sep 17 00:00:00 2001 From: Anna Savarin Date: Fri, 28 Jul 2017 11:50:11 +0200 Subject: [PATCH 6/6] [HDP-6774] Fixing failing tests by updating HtmlUnit driver dependency --- core/pom.xml | 2 +- pom.xml | 5 +++-- sql/hive-thriftserver/pom.xml | 2 +- streaming/pom.xml | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index bad3655452fb4..b6a1ac2475e4b 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -283,7 +283,7 @@ org.seleniumhq.selenium - selenium-htmlunit-driver + htmlunit-driver test diff --git a/pom.xml b/pom.xml index a66156c9050a2..48bdeaf427520 100644 --- a/pom.xml +++ b/pom.xml @@ -179,6 +179,7 @@ 4.5.3 1.1 2.52.0 + 2.27 2.8 1.8 1.0.0 @@ -466,8 +467,8 @@ org.seleniumhq.selenium - selenium-htmlunit-driver - ${selenium.version} + htmlunit-driver + ${selenium.htmlunitdriver.version} test diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 1abc0a253098c..87a9cae6677e7 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -71,7 +71,7 @@ org.seleniumhq.selenium - selenium-htmlunit-driver + htmlunit-driver test diff --git a/streaming/pom.xml b/streaming/pom.xml index 644fc50bf507b..e416f02ae23c0 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -109,7 +109,7 @@ org.seleniumhq.selenium - selenium-htmlunit-driver + htmlunit-driver test