From f069d38c8d3c0bfa91b70a60e4e556ec708fc411 Mon Sep 17 00:00:00 2001 From: Inigo Goiri Date: Fri, 19 Oct 2018 17:16:04 -0700 Subject: [PATCH] HDFS-13983. TestOfflineImageViewer crashes in windows. Contributed by Vinayakumar B. --- .../OfflineImageViewerPB.java | 31 +++++--- .../TestOfflineImageViewer.java | 76 ++++++++++--------- 2 files changed, 63 insertions(+), 44 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java index 34a85a6dce7..782ace818aa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java @@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.StringUtils; /** @@ -135,7 +136,7 @@ public class OfflineImageViewerPB { */ public static void main(String[] args) throws Exception { int status = run(args); - System.exit(status); + ExitUtil.terminate(status); } public static int run(String[] args) throws Exception { @@ -175,19 +176,24 @@ public class OfflineImageViewerPB { String tempPath = cmd.getOptionValue("t", ""); Configuration conf = new Configuration(); - try (PrintStream out = outputFile.equals("-") ? - System.out : new PrintStream(outputFile, "UTF-8")) { + PrintStream out = null; + try { + out = outputFile.equals("-") || "REVERSEXML".equalsIgnoreCase(processor) ? + System.out : new PrintStream(outputFile, "UTF-8"); switch (StringUtils.toUpperCase(processor)) { case "FILEDISTRIBUTION": long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0")); int step = Integer.parseInt(cmd.getOptionValue("step", "0")); boolean formatOutput = cmd.hasOption("format"); - new FileDistributionCalculator(conf, maxSize, step, formatOutput, out) - .visit(new RandomAccessFile(inputFile, "r")); + try (RandomAccessFile r = new RandomAccessFile(inputFile, "r")) { + new FileDistributionCalculator(conf, maxSize, step, formatOutput, out) + .visit(r); + } break; case "XML": - new PBImageXmlWriter(conf, out).visit(new RandomAccessFile(inputFile, - "r")); + try (RandomAccessFile r = new RandomAccessFile(inputFile, "r")) { + new PBImageXmlWriter(conf, out).visit(r); + } break; case "REVERSEXML": try { @@ -196,7 +202,7 @@ public class OfflineImageViewerPB { System.err.println("OfflineImageReconstructor failed: " + e.getMessage()); e.printStackTrace(System.err); - System.exit(1); + ExitUtil.terminate(1); } break; case "WEB": @@ -208,8 +214,9 @@ public class OfflineImageViewerPB { break; case "DELIMITED": try (PBImageDelimitedTextWriter writer = - new PBImageDelimitedTextWriter(out, delimiter, tempPath)) { - writer.visit(new RandomAccessFile(inputFile, "r")); + new PBImageDelimitedTextWriter(out, delimiter, tempPath); + RandomAccessFile r = new RandomAccessFile(inputFile, "r")) { + writer.visit(r); } break; default: @@ -223,6 +230,10 @@ public class OfflineImageViewerPB { } catch (IOException e) { System.err.println("Encountered exception. Exiting: " + e.getMessage()); e.printStackTrace(System.err); + } finally { + if (out != null && out != System.out) { + out.close(); + } } return -1; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index 16b35dc6a28..5574d7b4a94 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -355,8 +355,10 @@ public class TestOfflineImageViewer { File truncatedFile = new File(tempDir, "truncatedFsImage"); PrintStream output = new PrintStream(NullOutputStream.NULL_OUTPUT_STREAM); copyPartOfFile(originalFsimage, truncatedFile); - new FileDistributionCalculator(new Configuration(), 0, 0, false, output) - .visit(new RandomAccessFile(truncatedFile, "r")); + try (RandomAccessFile r = new RandomAccessFile(truncatedFile, "r")) { + new FileDistributionCalculator(new Configuration(), 0, 0, false, output) + .visit(r); + } } private void copyPartOfFile(File src, File dest) throws IOException { @@ -377,38 +379,41 @@ public class TestOfflineImageViewer { @Test public void testFileDistributionCalculator() throws IOException { - ByteArrayOutputStream output = new ByteArrayOutputStream(); - PrintStream o = new PrintStream(output); - new FileDistributionCalculator(new Configuration(), 0, 0, false, o) - .visit(new RandomAccessFile(originalFsimage, "r")); - o.close(); + try (ByteArrayOutputStream output = new ByteArrayOutputStream(); + PrintStream o = new PrintStream(output); + RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) { + new FileDistributionCalculator(new Configuration(), 0, 0, false, o) + .visit(r); + o.close(); - String outputString = output.toString(); - Pattern p = Pattern.compile("totalFiles = (\\d+)\n"); - Matcher matcher = p.matcher(outputString); - assertTrue(matcher.find() && matcher.groupCount() == 1); - int totalFiles = Integer.parseInt(matcher.group(1)); - assertEquals(NUM_DIRS * FILES_PER_DIR + filesECCount + 1, totalFiles); + String outputString = output.toString(); + Pattern p = Pattern.compile("totalFiles = (\\d+)\n"); + Matcher matcher = p.matcher(outputString); + assertTrue(matcher.find() && matcher.groupCount() == 1); + int totalFiles = Integer.parseInt(matcher.group(1)); + assertEquals(NUM_DIRS * FILES_PER_DIR + filesECCount + 1, totalFiles); - p = Pattern.compile("totalDirectories = (\\d+)\n"); - matcher = p.matcher(outputString); - assertTrue(matcher.find() && matcher.groupCount() == 1); - int totalDirs = Integer.parseInt(matcher.group(1)); - // totalDirs includes root directory - assertEquals(dirCount + 1, totalDirs); + p = Pattern.compile("totalDirectories = (\\d+)\n"); + matcher = p.matcher(outputString); + assertTrue(matcher.find() && matcher.groupCount() == 1); + int totalDirs = Integer.parseInt(matcher.group(1)); + // totalDirs includes root directory + assertEquals(dirCount + 1, totalDirs); - FileStatus maxFile = Collections.max(writtenFiles.values(), - new Comparator() { - @Override - public int compare(FileStatus first, FileStatus second) { - return first.getLen() < second.getLen() ? -1 : - ((first.getLen() == second.getLen()) ? 0 : 1); - } - }); - p = Pattern.compile("maxFileSize = (\\d+)\n"); - matcher = p.matcher(output.toString("UTF-8")); - assertTrue(matcher.find() && matcher.groupCount() == 1); - assertEquals(maxFile.getLen(), Long.parseLong(matcher.group(1))); + FileStatus maxFile = Collections.max(writtenFiles.values(), + new Comparator() { + @Override + public int compare(FileStatus first, FileStatus second) { + return first.getLen() < second.getLen() ? + -1 : + ((first.getLen() == second.getLen()) ? 0 : 1); + } + }); + p = Pattern.compile("maxFileSize = (\\d+)\n"); + matcher = p.matcher(output.toString("UTF-8")); + assertTrue(matcher.find() && matcher.groupCount() == 1); + assertEquals(maxFile.getLen(), Long.parseLong(matcher.group(1))); + } } @Test @@ -514,7 +519,9 @@ public class TestOfflineImageViewer { ByteArrayOutputStream output = new ByteArrayOutputStream(); PrintStream o = new PrintStream(output); PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), o); - v.visit(new RandomAccessFile(originalFsimage, "r")); + try (RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) { + v.visit(r); + } SAXParserFactory spf = SAXParserFactory.newInstance(); SAXParser parser = spf.newSAXParser(); final String xml = output.toString(); @@ -667,10 +674,11 @@ public class TestOfflineImageViewer { final String DELIMITER = "\t"; ByteArrayOutputStream output = new ByteArrayOutputStream(); - try (PrintStream o = new PrintStream(output)) { + try (PrintStream o = new PrintStream(output); + RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) { PBImageDelimitedTextWriter v = new PBImageDelimitedTextWriter(o, DELIMITER, db); - v.visit(new RandomAccessFile(originalFsimage, "r")); + v.visit(r); } Set fileNames = new HashSet<>();