Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-13983. TestOfflineImageViewer crashes in windows. Contributed by Vinayakumar B.

This commit is contained in:
Inigo Goiri 2018-10-19 17:16:04 -07:00
parent e2a9fa8448
commit f069d38c8d
2 changed files with 63 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
View File

@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.StringUtils;
/** /**
@ -135,7 +136,7 @@ public class OfflineImageViewerPB {
*/ */
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
int status = run(args); int status = run(args);
System.exit(status); ExitUtil.terminate(status);
} }
public static int run(String[] args) throws Exception { public static int run(String[] args) throws Exception {
@ -175,19 +176,24 @@ public class OfflineImageViewerPB {
String tempPath = cmd.getOptionValue("t", ""); String tempPath = cmd.getOptionValue("t", "");
Configuration conf = new Configuration(); Configuration conf = new Configuration();
try (PrintStream out = outputFile.equals("-") ? PrintStream out = null;
System.out : new PrintStream(outputFile, "UTF-8")) { try {
out = outputFile.equals("-") || "REVERSEXML".equalsIgnoreCase(processor) ?
System.out : new PrintStream(outputFile, "UTF-8");
switch (StringUtils.toUpperCase(processor)) { switch (StringUtils.toUpperCase(processor)) {
case "FILEDISTRIBUTION": case "FILEDISTRIBUTION":
long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0")); long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0"));
int step = Integer.parseInt(cmd.getOptionValue("step", "0")); int step = Integer.parseInt(cmd.getOptionValue("step", "0"));
boolean formatOutput = cmd.hasOption("format"); boolean formatOutput = cmd.hasOption("format");
new FileDistributionCalculator(conf, maxSize, step, formatOutput, out) try (RandomAccessFile r = new RandomAccessFile(inputFile, "r")) {
.visit(new RandomAccessFile(inputFile, "r")); new FileDistributionCalculator(conf, maxSize, step, formatOutput, out)
.visit(r);
}
break; break;
case "XML": case "XML":
new PBImageXmlWriter(conf, out).visit(new RandomAccessFile(inputFile, try (RandomAccessFile r = new RandomAccessFile(inputFile, "r")) {
"r")); new PBImageXmlWriter(conf, out).visit(r);
}
break; break;
case "REVERSEXML": case "REVERSEXML":
try { try {
@ -196,7 +202,7 @@ public class OfflineImageViewerPB {
System.err.println("OfflineImageReconstructor failed: " System.err.println("OfflineImageReconstructor failed: "
+ e.getMessage()); + e.getMessage());
e.printStackTrace(System.err); e.printStackTrace(System.err);
System.exit(1); ExitUtil.terminate(1);
} }
break; break;
case "WEB": case "WEB":
@ -208,8 +214,9 @@ public class OfflineImageViewerPB {
break; break;
case "DELIMITED": case "DELIMITED":
try (PBImageDelimitedTextWriter writer = try (PBImageDelimitedTextWriter writer =
new PBImageDelimitedTextWriter(out, delimiter, tempPath)) { new PBImageDelimitedTextWriter(out, delimiter, tempPath);
writer.visit(new RandomAccessFile(inputFile, "r")); RandomAccessFile r = new RandomAccessFile(inputFile, "r")) {
writer.visit(r);
} }
break; break;
default: default:
@ -223,6 +230,10 @@ public class OfflineImageViewerPB {
} catch (IOException e) { } catch (IOException e) {
System.err.println("Encountered exception. Exiting: " + e.getMessage()); System.err.println("Encountered exception. Exiting: " + e.getMessage());
e.printStackTrace(System.err); e.printStackTrace(System.err);
} finally {
if (out != null && out != System.out) {
out.close();
}
} }
return -1; return -1;
} }

76
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
View File

@ -355,8 +355,10 @@ public class TestOfflineImageViewer {
File truncatedFile = new File(tempDir, "truncatedFsImage"); File truncatedFile = new File(tempDir, "truncatedFsImage");
PrintStream output = new PrintStream(NullOutputStream.NULL_OUTPUT_STREAM); PrintStream output = new PrintStream(NullOutputStream.NULL_OUTPUT_STREAM);
copyPartOfFile(originalFsimage, truncatedFile); copyPartOfFile(originalFsimage, truncatedFile);
new FileDistributionCalculator(new Configuration(), 0, 0, false, output) try (RandomAccessFile r = new RandomAccessFile(truncatedFile, "r")) {
.visit(new RandomAccessFile(truncatedFile, "r")); new FileDistributionCalculator(new Configuration(), 0, 0, false, output)
.visit(r);
}
} }
private void copyPartOfFile(File src, File dest) throws IOException { private void copyPartOfFile(File src, File dest) throws IOException {
@ -377,38 +379,41 @@ public class TestOfflineImageViewer {
@Test @Test
public void testFileDistributionCalculator() throws IOException { public void testFileDistributionCalculator() throws IOException {
ByteArrayOutputStream output = new ByteArrayOutputStream(); try (ByteArrayOutputStream output = new ByteArrayOutputStream();
PrintStream o = new PrintStream(output); PrintStream o = new PrintStream(output);
new FileDistributionCalculator(new Configuration(), 0, 0, false, o) RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) {
.visit(new RandomAccessFile(originalFsimage, "r")); new FileDistributionCalculator(new Configuration(), 0, 0, false, o)
o.close(); .visit(r);
o.close();
String outputString = output.toString(); String outputString = output.toString();
Pattern p = Pattern.compile("totalFiles = (\\d+)\n"); Pattern p = Pattern.compile("totalFiles = (\\d+)\n");
Matcher matcher = p.matcher(outputString); Matcher matcher = p.matcher(outputString);
assertTrue(matcher.find() && matcher.groupCount() == 1); assertTrue(matcher.find() && matcher.groupCount() == 1);
int totalFiles = Integer.parseInt(matcher.group(1)); int totalFiles = Integer.parseInt(matcher.group(1));
assertEquals(NUM_DIRS * FILES_PER_DIR + filesECCount + 1, totalFiles); assertEquals(NUM_DIRS * FILES_PER_DIR + filesECCount + 1, totalFiles);
p = Pattern.compile("totalDirectories = (\\d+)\n"); p = Pattern.compile("totalDirectories = (\\d+)\n");
matcher = p.matcher(outputString); matcher = p.matcher(outputString);
assertTrue(matcher.find() && matcher.groupCount() == 1); assertTrue(matcher.find() && matcher.groupCount() == 1);
int totalDirs = Integer.parseInt(matcher.group(1)); int totalDirs = Integer.parseInt(matcher.group(1));
// totalDirs includes root directory // totalDirs includes root directory
assertEquals(dirCount + 1, totalDirs); assertEquals(dirCount + 1, totalDirs);
FileStatus maxFile = Collections.max(writtenFiles.values(), FileStatus maxFile = Collections.max(writtenFiles.values(),
new Comparator<FileStatus>() { new Comparator<FileStatus>() {
@Override @Override
public int compare(FileStatus first, FileStatus second) { public int compare(FileStatus first, FileStatus second) {
return first.getLen() < second.getLen() ? -1 : return first.getLen() < second.getLen() ?
((first.getLen() == second.getLen()) ? 0 : 1); -1 :
} ((first.getLen() == second.getLen()) ? 0 : 1);
}); }
p = Pattern.compile("maxFileSize = (\\d+)\n"); });
matcher = p.matcher(output.toString("UTF-8")); p = Pattern.compile("maxFileSize = (\\d+)\n");
assertTrue(matcher.find() && matcher.groupCount() == 1); matcher = p.matcher(output.toString("UTF-8"));
assertEquals(maxFile.getLen(), Long.parseLong(matcher.group(1))); assertTrue(matcher.find() && matcher.groupCount() == 1);
assertEquals(maxFile.getLen(), Long.parseLong(matcher.group(1)));
}
} }
@Test @Test
@ -514,7 +519,9 @@ public class TestOfflineImageViewer {
ByteArrayOutputStream output = new ByteArrayOutputStream(); ByteArrayOutputStream output = new ByteArrayOutputStream();
PrintStream o = new PrintStream(output); PrintStream o = new PrintStream(output);
PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), o); PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), o);
v.visit(new RandomAccessFile(originalFsimage, "r")); try (RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) {
v.visit(r);
}
SAXParserFactory spf = SAXParserFactory.newInstance(); SAXParserFactory spf = SAXParserFactory.newInstance();
SAXParser parser = spf.newSAXParser(); SAXParser parser = spf.newSAXParser();
final String xml = output.toString(); final String xml = output.toString();
@ -667,10 +674,11 @@ public class TestOfflineImageViewer {
final String DELIMITER = "\t"; final String DELIMITER = "\t";
ByteArrayOutputStream output = new ByteArrayOutputStream(); ByteArrayOutputStream output = new ByteArrayOutputStream();
try (PrintStream o = new PrintStream(output)) { try (PrintStream o = new PrintStream(output);
RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) {
PBImageDelimitedTextWriter v = PBImageDelimitedTextWriter v =
new PBImageDelimitedTextWriter(o, DELIMITER, db); new PBImageDelimitedTextWriter(o, DELIMITER, db);
v.visit(new RandomAccessFile(originalFsimage, "r")); v.visit(r);
} }
Set<String> fileNames = new HashSet<>(); Set<String> fileNames = new HashSet<>();