diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d106509f359..8aad41d8a8b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -463,6 +463,9 @@ Release 2.0.3-alpha - Unreleased HDFS-4344. dfshealth.jsp throws NumberFormatException when dfs.hosts/dfs.hosts.exclude includes port number. (Andy Isaacson via atm) + HDFS-4468. Use the new StringUtils methods added by HADOOP-9252 and fix + TestHDFSCLI and TestQuota. (szetszwo) + BREAKDOWN OF HDFS-3077 SUBTASKS HDFS-3077. Quorum-based protocol for reading and writing edit logs. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 9564c543285..613f8defd17 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -923,6 +923,11 @@ public class DFSUtil { return capacity <= 0 ? 0 : (remaining * 100.0f)/capacity; } + /** Convert percentage to a string. */ + public static String percent2String(double percentage) { + return StringUtils.format("%.2f%%", percentage); + } + /** * Round bytes to GiB (gibibyte) * @param bytes number of bytes diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DSQuotaExceededException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DSQuotaExceededException.java index c7b22f7ac24..481c1305e8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DSQuotaExceededException.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DSQuotaExceededException.java @@ -20,7 +20,7 @@ package org.apache.hadoop.hdfs.protocol; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.util.StringUtils; +import static org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix.long2String; @InterfaceAudience.Private @InterfaceStability.Evolving @@ -41,9 +41,9 @@ public class DSQuotaExceededException extends QuotaExceededException { public String getMessage() { String msg = super.getMessage(); if (msg == null) { - return "The DiskSpace quota" + (pathName==null?"":(" of " + pathName)) + - " is exceeded: quota=" + StringUtils.humanReadableInt(quota) + - " diskspace consumed=" + StringUtils.humanReadableInt(count); + return "The DiskSpace quota" + (pathName==null?"": " of " + pathName) + + " is exceeded: quota = " + quota + " B = " + long2String(quota, "B", 2) + + " but diskspace consumed = " + count + " B = " + long2String(count, "B", 2); } else { return msg; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java index c62faea14a1..797c9a49f5e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java @@ -17,10 +17,13 @@ */ package org.apache.hadoop.hdfs.protocol; +import static org.apache.hadoop.hdfs.DFSUtil.percent2String; + import java.util.Date; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.NetworkTopology; @@ -242,8 +245,8 @@ public class DatanodeInfo extends DatanodeID implements Node { buffer.append("DFS Used: "+u+" ("+StringUtils.byteDesc(u)+")"+"\n"); buffer.append("Non DFS Used: "+nonDFSUsed+" ("+StringUtils.byteDesc(nonDFSUsed)+")"+"\n"); buffer.append("DFS Remaining: " +r+ " ("+StringUtils.byteDesc(r)+")"+"\n"); - buffer.append("DFS Used%: "+StringUtils.limitDecimalTo2(usedPercent)+"%\n"); - buffer.append("DFS Remaining%: "+StringUtils.limitDecimalTo2(remainingPercent)+"%\n"); + buffer.append("DFS Used%: "+percent2String(usedPercent) + "\n"); + buffer.append("DFS Remaining%: "+percent2String(remainingPercent) + "\n"); buffer.append("Last contact: "+new Date(lastUpdate)+"\n"); return buffer.toString(); } @@ -267,7 +270,7 @@ public class DatanodeInfo extends DatanodeID implements Node { } buffer.append(" " + c + "(" + StringUtils.byteDesc(c)+")"); buffer.append(" " + u + "(" + StringUtils.byteDesc(u)+")"); - buffer.append(" " + StringUtils.limitDecimalTo2(((1.0*u)/c)*100)+"%"); + buffer.append(" " + percent2String(u/(double)c)); buffer.append(" " + r + "(" + StringUtils.byteDesc(r)+")"); buffer.append(" " + new Date(lastUpdate)); return buffer.toString(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java index 1b3db818d15..0f0a989f8c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ClusterJspHelper.java @@ -569,12 +569,10 @@ class ClusterJspHelper { toXmlItemBlock(doc, "DFS Remaining", StringUtils.byteDesc(free)); // dfsUsedPercent - toXmlItemBlock(doc, "DFS Used%", - StringUtils.limitDecimalTo2(dfsUsedPercent)+ "%"); + toXmlItemBlock(doc, "DFS Used%", DFSUtil.percent2String(dfsUsedPercent)); // dfsRemainingPercent - toXmlItemBlock(doc, "DFS Remaining%", - StringUtils.limitDecimalTo2(dfsRemainingPercent) + "%"); + toXmlItemBlock(doc, "DFS Remaining%", DFSUtil.percent2String(dfsRemainingPercent)); doc.endTag(); // storage diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java index c6abb6faddd..9c9a1cc8fa5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import static org.apache.hadoop.hdfs.DFSUtil.percent2String; + import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; @@ -64,6 +66,14 @@ import org.znerd.xmlenc.XMLOutputter; import com.google.common.base.Preconditions; class NamenodeJspHelper { + static String fraction2String(double value) { + return StringUtils.format("%.2f", value); + } + + static String fraction2String(long numerator, long denominator) { + return fraction2String(numerator/(double)denominator); + } + static String getSafeModeText(FSNamesystem fsn) { if (!fsn.isInSafeMode()) return ""; @@ -361,20 +371,20 @@ class NamenodeJspHelper { + "DFS Remaining" + colTxt() + ":" + colTxt() + StringUtils.byteDesc(remaining) + rowTxt() + colTxt() + "DFS Used%" + colTxt() + ":" + colTxt() - + StringUtils.limitDecimalTo2(percentUsed) + " %" + rowTxt() + + percent2String(percentUsed) + rowTxt() + colTxt() + "DFS Remaining%" + colTxt() + ":" + colTxt() - + StringUtils.limitDecimalTo2(percentRemaining) + " %" + + percent2String(percentRemaining) + rowTxt() + colTxt() + "Block Pool Used" + colTxt() + ":" + colTxt() + StringUtils.byteDesc(bpUsed) + rowTxt() + colTxt() + "Block Pool Used%"+ colTxt() + ":" + colTxt() - + StringUtils.limitDecimalTo2(percentBpUsed) + " %" + + percent2String(percentBpUsed) + rowTxt() + colTxt() + "DataNodes usages" + colTxt() + ":" + colTxt() + "Min %" + colTxt() + "Median %" + colTxt() + "Max %" + colTxt() + "stdev %" + rowTxt() + colTxt() + colTxt() + colTxt() - + StringUtils.limitDecimalTo2(min) + " %" - + colTxt() + StringUtils.limitDecimalTo2(median) + " %" - + colTxt() + StringUtils.limitDecimalTo2(max) + " %" - + colTxt() + StringUtils.limitDecimalTo2(dev) + " %" + + percent2String(min) + + colTxt() + percent2String(median) + + colTxt() + percent2String(max) + + colTxt() + percent2String(dev) + rowTxt() + colTxt() + "Live Nodes " + colTxt() + ":" + colTxt() + live.size() @@ -562,9 +572,9 @@ class NamenodeJspHelper { long u = d.getDfsUsed(); long nu = d.getNonDfsUsed(); long r = d.getRemaining(); - String percentUsed = StringUtils.limitDecimalTo2(d.getDfsUsedPercent()); - String percentRemaining = StringUtils.limitDecimalTo2(d - .getRemainingPercent()); + final double percentUsedValue = d.getDfsUsedPercent(); + String percentUsed = fraction2String(percentUsedValue); + String percentRemaining = fraction2String(d.getRemainingPercent()); String adminState = d.getAdminState().toString(); @@ -572,32 +582,30 @@ class NamenodeJspHelper { long currentTime = Time.now(); long bpUsed = d.getBlockPoolUsed(); - String percentBpUsed = StringUtils.limitDecimalTo2(d - .getBlockPoolUsedPercent()); + String percentBpUsed = fraction2String(d.getBlockPoolUsedPercent()); out.print(" " + ((currentTime - timestamp) / 1000) + "" + adminState + "" - + StringUtils.limitDecimalTo2(c * 1.0 / diskBytes) + + fraction2String(c, diskBytes) + "" - + StringUtils.limitDecimalTo2(u * 1.0 / diskBytes) + + fraction2String(u, diskBytes) + "" - + StringUtils.limitDecimalTo2(nu * 1.0 / diskBytes) + + fraction2String(nu, diskBytes) + "" - + StringUtils.limitDecimalTo2(r * 1.0 / diskBytes) + + fraction2String(r, diskBytes) + "" + percentUsed + "" - + ServletUtil.percentageGraph((int) Double.parseDouble(percentUsed), - 100) + + ServletUtil.percentageGraph((int)percentUsedValue, 100) + "" + percentRemaining + "" + d.numBlocks()+"\n" + "" - + StringUtils.limitDecimalTo2(bpUsed * 1.0 / diskBytes) + + fraction2String(bpUsed, diskBytes) + "" + percentBpUsed + "" diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index b3c9309054d..7db2b5614ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -316,8 +316,7 @@ public class DFSAdmin extends FsShell { System.out.println("DFS Used: " + used + " (" + StringUtils.byteDesc(used) + ")"); System.out.println("DFS Used%: " - + StringUtils.limitDecimalTo2(((1.0 * used) / presentCapacity) * 100) - + "%"); + + StringUtils.formatPercent(used/(double)presentCapacity, 2)); /* These counts are not always upto date. They are updated after * iteration of an internal list. Should be updated in a few seconds to diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java index f6e09283224..a1886130816 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java @@ -68,8 +68,8 @@ public class TestQuota { throw new DSQuotaExceededException(bytes, bytes); } catch(DSQuotaExceededException e) { - assertEquals("The DiskSpace quota is exceeded: quota=1.0k " + - "diskspace consumed=1.0k", e.getMessage()); + assertEquals("The DiskSpace quota is exceeded: quota = 1024 B = 1 KB" + + " but diskspace consumed = 1024 B = 1 KB", e.getMessage()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index 23498564749..a71b3fedb9f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -1182,7 +1182,7 @@ RegexpComparator - ^1\.0k\s+hdfs:///dir0/data1k + ^1\.0 K\s+hdfs:///dir0/data1k @@ -15590,7 +15590,7 @@ RegexpComparator - put: The DiskSpace quota of /dir1 is exceeded: quota=1.0k diskspace consumed=[0-9.]+[kmg]* + put: The DiskSpace quota of /dir1 is exceeded: quota = 1024 B = 1 KB but diskspace consumed = [0-9]+ B = [0-9.]+ [KMG]B*