From 379d587d23e3499782125addb26a19cde33618ec Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Sat, 30 Jul 2016 08:28:33 +0900 Subject: [PATCH] HDFS-10691. FileDistribution fails in hdfs oiv command due to ArrayIndexOutOfBoundsException. Contributed by Yiqun Lin. --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../FileDistributionCalculator.java | 6 +++ .../FileDistributionVisitor.java | 4 ++ .../TestOfflineImageViewer.java | 53 +++++++++++++++++++ 4 files changed, 66 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a38d3b208a5..9efd8a4caaf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -15,6 +15,9 @@ Release 2.7.4 - UNRELEASED HDFS-9601. NNThroughputBenchmark.BlockReportStats should handle NotReplicatedYetException on adding block. (iwasakims) + HDFS-10691. FileDistribution fails in hdfs oiv command due to + ArrayIndexOutOfBoundsException. (Yiqun Lin via aajisaka) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java index 056ad96a5ba..33ab641e54c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java @@ -128,6 +128,12 @@ final class FileDistributionCalculator { int bucket = fileSize > maxSize ? distribution.length - 1 : (int) Math .ceil((double)fileSize / steps); + // Compare the bucket value with distribution's length again, + // because sometimes the bucket value will be equal to + // the length when maxSize can't be divided completely by step. + if (bucket >= distribution.length) { + bucket = distribution.length - 1; + } ++distribution[bucket]; } else if (p.getType() == INodeSection.INode.Type.DIRECTORY) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java index 146d00a85ee..1cef720624d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java @@ -145,6 +145,10 @@ class FileDistributionVisitor extends TextWriterImageVisitor { high = distribution.length-1; else high = (int)Math.ceil((double)current.fileSize / step); + + if (high >= distribution.length) { + high = distribution.length - 1; + } distribution[high]++; if(totalFiles % 1000000 == 1) System.out.println("Files processed: " + totalFiles diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index a8d1d544501..04eecf2f411 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -58,6 +58,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; @@ -399,4 +400,56 @@ public class TestOfflineImageViewer { connection.connect(); assertEquals(expectedCode, connection.getResponseCode()); } + + @Test + public void testFileDistributionCalculatorForException() throws Exception { + File fsimageFile = null; + MiniDFSCluster cluster = null; + Configuration conf = new Configuration(); + HashMap files = Maps.newHashMap(); + + // Create a initial fsimage file + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster.waitActive(); + DistributedFileSystem hdfs = cluster.getFileSystem(); + + // Create a reasonable namespace + Path dir = new Path("/dir"); + hdfs.mkdirs(dir); + files.put(dir.toString(), pathToFileEntry(hdfs, dir.toString())); + // Create files with byte size that can't be divided by step size, + // the byte size for here are 3, 9, 15, 21. + for (int i = 0; i < FILES_PER_DIR; i++) { + Path file = new Path(dir, "file" + i); + DFSTestUtil.createFile(hdfs, file, 6 * i + 3, (short) 1, 0); + + files.put(file.toString(), pathToFileEntry(hdfs, file.toString())); + } + + // Write results to the fsimage file + hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER, false); + hdfs.saveNamespace(); + // Determine location of fsimage file + fsimageFile = + FSImageTestUtil.findLatestImageFile(FSImageTestUtil + .getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0)); + if (fsimageFile == null) { + throw new RuntimeException("Didn't generate or can't find fsimage"); + } + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + + // Run the test with params -maxSize 23 and -step 4, it will not throw + // ArrayIndexOutOfBoundsException with index 6 when deals with + // 21 byte size file. + int status = + OfflineImageViewerPB.run(new String[] {"-i", + fsimageFile.getAbsolutePath(), "-o", "-", "-p", + "FileDistribution", "-maxSize", "23", "-step", "4"}); + assertEquals(0, status); + } }