From 6e416a83d1e674ecd018d1db74a2d88e738deb40 Mon Sep 17 00:00:00 2001 From: Ayush Saxena Date: Sun, 17 May 2020 03:06:27 +0530 Subject: [PATCH] HDFS-15358. RBF: Unify router datanode UI with namenode datanode UI. Contributed by Ayush Saxena. --- .../main/webapps/router/federationhealth.html | 70 +++++++++- .../main/webapps/router/federationhealth.js | 122 ++++++++++++++++-- .../src/main/webapps/hdfs/dfshealth.html | 2 + 3 files changed, 179 insertions(+), 15 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html index 534dd95794d..a9afe401fcb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html @@ -23,6 +23,11 @@ Router Information + @@ -315,29 +320,44 @@ + +

Datanode usage histogram

+

In operation

+

DataNode State

+ + + - + - + {#LiveNodes} + - + + + - + - + {/LiveNodes} {#DeadNodes} + + - - + + + + + {/DeadNodes}
State NodeHttp Address Last contactLast Block Report Used Non DFS Used CapacityBlocks Block pool usedVersion
{state} {location}/{name} ({xferaddr}){#helper_relative_time value="{lastContact}"/}{dnWebAddress}{lastContact}s{lastBlockReport}m {used|fmt_bytes} {nonDfsUsedSpace|fmt_bytes} @@ -349,26 +369,58 @@ {numBlocks} {blockPoolUsed|fmt_bytes} ({blockPoolUsedPercent|fmt_percentage}){version}
{state} {location}/{name} ({xferaddr}) {#helper_relative_time value="{lastContact}"/}
+

Entering Maintenance

+ + {?EnteringMaintenanceNodes} + + + + + + + + + + {#EnteringMaintenanceNodes} + + + + + + + {/EnteringMaintenanceNodes} +
NodeUnder replicated blocksBlocks with no live replicasUnder Replicated Blocks
In files under construction
{name} ({xferaddr}){underReplicatedBlocks}{maintenanceOnlyReplicas}{underReplicateInOpenFiles}
+ {:else} + No nodes are entering maintenance. + {/EnteringMaintenanceNodes} + +

Decommissioning

+{?DecomNodes} @@ -387,6 +439,9 @@ {/DecomNodes}
+{:else} +No nodes are decommissioning. +{/DecomNodes} @@ -449,6 +504,7 @@ + diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js index b7b2ba31c6c..86eda24540c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js @@ -278,20 +278,34 @@ for (var i = 0, e = nodes.length; i < e; ++i) { var n = nodes[i]; n.usedPercentage = Math.round((n.used + n.nonDfsUsedSpace) * 1.0 / n.capacity * 100); + var port = n.infoAddr.split(":")[1]; + var securePort = n.infoSecureAddr.split(":")[1]; + var dnHost = n.name.split(":")[0]; + n.dnWebAddress = "http://" + dnHost + ":" + port; + if (securePort != 0) { + n.dnWebAddress = "https://" + dnHost + ":" + securePort; + } + if (n.adminState === "In Service") { n.state = "alive"; } else if (nodes[i].adminState === "Decommission In Progress") { n.state = "decommissioning"; } else if (nodes[i].adminState === "Decommissioned") { n.state = "decommissioned"; + } else if (nodes[i].adminState === "Entering Maintenance") { + n.state = "entering-maintenance"; + } else if (nodes[i].adminState === "In Maintenance") { + n.state = "in-maintenance"; } } } function augment_dead_nodes(nodes) { for (var i = 0, e = nodes.length; i < e; ++i) { - if (nodes[i].decommissioned) { + if (nodes[i].adminState === "Decommissioned") { nodes[i].state = "down-decommissioned"; + } else if (nodes[i].adminState === "In Maintenance") { + nodes[i].state = "down-maintenance"; } else { nodes[i].state = "down"; } @@ -303,9 +317,77 @@ r.DeadNodes = node_map_to_array(JSON.parse(r.DeadNodes)); augment_dead_nodes(r.DeadNodes); r.DecomNodes = node_map_to_array(JSON.parse(r.DecomNodes)); + r.EnteringMaintenanceNodes = node_map_to_array(JSON.parse(r.EnteringMaintenanceNodes)); return r; } + function renderHistogram(dnData) { + var data = dnData.LiveNodes.map(function(dn) { + return (dn.usedSpace / dn.capacity) * 100.0; + }); + + var formatCount = d3.format(",.0f"); + + var widthCap = $("div.container").width(); + var heightCap = 150; + + var margin = {top: 10, right: 60, bottom: 30, left: 30}, + width = widthCap * 0.9, + height = heightCap - margin.top - margin.bottom; + + var x = d3.scaleLinear() + .domain([0.0, 100.0]) + .range([0, width]); + + var bins = d3.histogram() + .domain(x.domain()) + .thresholds(x.ticks(20)) + (data); + + var y = d3.scaleLinear() + .domain([0, d3.max(bins, function(d) { return d.length; })]) + .range([height, 0]); + + var svg = d3.select("#datanode-usage-histogram").append("svg") + .attr("width", width + 50.0) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + svg.append("text") + .attr("x", (width / 2)) + .attr("y", heightCap - 6 - (margin.top / 2)) + .attr("text-anchor", "middle") + .style("font-size", "15px") + .text("Disk usage of each DataNode (%)"); + + var bar = svg.selectAll(".bar") + .data(bins) + .enter().append("g") + .attr("class", "bar") + .attr("transform", function(d) { return "translate(" + x(d.x0) + "," + y(d.length) + ")"; }); + + window.liveNodes = dnData.LiveNodes; + + bar.append("rect") + .attr("x", 1) + .attr("width", x(bins[0].x1) - x(bins[0].x0) - 1) + .attr("height", function(d) { return height - y(d.length); }) + .attr("onclick", function (d) { return "open_hostip_list(" + d.x0 + "," + d.x1 + ")"; }); + + bar.append("text") + .attr("dy", ".75em") + .attr("y", 6) + .attr("x", (x(bins[0].x1) - x(bins[0].x0)) / 2) + .attr("text-anchor", "middle") + .text(function(d) { return formatCount(d.length); }); + + svg.append("g") + .attr("class", "axis axis--x") + .attr("transform", "translate(0," + height + ")") + .call(d3.axisBottom(x)); + } + $.get( 'jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo', guard_with_startup_progress(function (resp) { @@ -315,14 +397,38 @@ $('#tab-datanode').html(out); $('#table-datanodes').dataTable( { 'lengthMenu': [ [25, 50, 100, -1], [25, 50, 100, "All"] ], + 'columnDefs': [ + { 'targets': [ 0 ], 'visible': false, 'searchable': false } + ], 'columns': [ - { 'orderDataType': 'ng-value', 'searchable': true }, - { 'orderDataType': 'ng-value', 'type': 'numeric' }, - { 'orderDataType': 'ng-value', 'type': 'numeric' }, - { 'orderDataType': 'ng-value', 'type': 'numeric' }, - { 'orderDataType': 'ng-value', 'type': 'numeric' }, - { 'orderDataType': 'ng-value', 'type': 'numeric'} - ]}); + { 'orderDataType': 'ng-value', 'searchable': true , "defaultContent": "" }, + { 'orderDataType': 'ng-value', 'searchable': true , "defaultContent": "" }, + { 'orderDataType': 'ng-value', 'searchable': true , "defaultContent": ""}, + { 'orderDataType': 'ng-value', 'type': 'num' , "defaultContent": 0}, + { 'orderDataType': 'ng-value', 'type': 'num' , "defaultContent": 0}, + { 'orderDataType': 'ng-value', 'type': 'num' , "defaultContent": 0}, + { 'orderDataType': 'ng-value', 'type': 'num' , "defaultContent": 0}, + { 'orderDataType': 'ng-value', 'type': 'num' , "defaultContent": 0}, + { 'type': 'num' , "defaultContent": 0}, + { 'orderDataType': 'ng-value', 'type': 'num' , "defaultContent": 0}, + { 'type': 'string' , "defaultContent": ""} + ], + initComplete: function () { + var column = this.api().column([0]); + var select = $('') + .appendTo('#datanodefilter') + .on('change', function () { + var val = $.fn.dataTable.util.escapeRegex( + $(this).val()); + column.search(val ? '^' + val + '$' : '', true, false).draw(); + }); + console.log(select); + column.data().unique().sort().each(function (d, j) { + select.append(''); + }); + } + }); + renderHistogram(data); $('#ui-tabs a[href="#tab-datanode"]').tab('show'); }); })).fail(ajax_error_handler); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index 582420e1710..fcf4e0d0586 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -371,6 +371,8 @@ + + > {/DeadNodes}