Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-13983. TestOfflineImageViewer crashes in windows. Contributed by Vinayakumar B.

This commit is contained in:
Inigo Goiri 2018-10-19 17:16:04 -07:00
parent e2a9fa8448
commit f069d38c8d
2 changed files with 63 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
View File

@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.StringUtils;
/**
@ -135,7 +136,7 @@ public class OfflineImageViewerPB {
*/
public static void main(String[] args) throws Exception {
int status = run(args);
System.exit(status);
ExitUtil.terminate(status);
}
public static int run(String[] args) throws Exception {
@ -175,19 +176,24 @@ public class OfflineImageViewerPB {
String tempPath = cmd.getOptionValue("t", "");
Configuration conf = new Configuration();
try (PrintStream out = outputFile.equals("-") ?
System.out : new PrintStream(outputFile, "UTF-8")) {
PrintStream out = null;
try {
out = outputFile.equals("-") || "REVERSEXML".equalsIgnoreCase(processor) ?
System.out : new PrintStream(outputFile, "UTF-8");
switch (StringUtils.toUpperCase(processor)) {
case "FILEDISTRIBUTION":
long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0"));
int step = Integer.parseInt(cmd.getOptionValue("step", "0"));
boolean formatOutput = cmd.hasOption("format");
try (RandomAccessFile r = new RandomAccessFile(inputFile, "r")) {
new FileDistributionCalculator(conf, maxSize, step, formatOutput, out)
.visit(new RandomAccessFile(inputFile, "r"));
.visit(r);
}
break;
case "XML":
new PBImageXmlWriter(conf, out).visit(new RandomAccessFile(inputFile,
"r"));
try (RandomAccessFile r = new RandomAccessFile(inputFile, "r")) {
new PBImageXmlWriter(conf, out).visit(r);
}
break;
case "REVERSEXML":
try {
@ -196,7 +202,7 @@ public class OfflineImageViewerPB {
System.err.println("OfflineImageReconstructor failed: "
+ e.getMessage());
e.printStackTrace(System.err);
System.exit(1);
ExitUtil.terminate(1);
}
break;
case "WEB":
@ -208,8 +214,9 @@ public class OfflineImageViewerPB {
break;
case "DELIMITED":
try (PBImageDelimitedTextWriter writer =
new PBImageDelimitedTextWriter(out, delimiter, tempPath)) {
writer.visit(new RandomAccessFile(inputFile, "r"));
new PBImageDelimitedTextWriter(out, delimiter, tempPath);
RandomAccessFile r = new RandomAccessFile(inputFile, "r")) {
writer.visit(r);
}
break;
default:
@ -223,6 +230,10 @@ public class OfflineImageViewerPB {
} catch (IOException e) {
System.err.println("Encountered exception. Exiting: " + e.getMessage());
e.printStackTrace(System.err);
} finally {
if (out != null && out != System.out) {
out.close();
}
}
return -1;
}

22
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
View File

@ -355,8 +355,10 @@ public class TestOfflineImageViewer {
File truncatedFile = new File(tempDir, "truncatedFsImage");
PrintStream output = new PrintStream(NullOutputStream.NULL_OUTPUT_STREAM);
copyPartOfFile(originalFsimage, truncatedFile);
try (RandomAccessFile r = new RandomAccessFile(truncatedFile, "r")) {
new FileDistributionCalculator(new Configuration(), 0, 0, false, output)
.visit(new RandomAccessFile(truncatedFile, "r"));
.visit(r);
}
}
private void copyPartOfFile(File src, File dest) throws IOException {
@ -377,10 +379,11 @@ public class TestOfflineImageViewer {
@Test
public void testFileDistributionCalculator() throws IOException {
ByteArrayOutputStream output = new ByteArrayOutputStream();
try (ByteArrayOutputStream output = new ByteArrayOutputStream();
PrintStream o = new PrintStream(output);
RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) {
new FileDistributionCalculator(new Configuration(), 0, 0, false, o)
.visit(new RandomAccessFile(originalFsimage, "r"));
.visit(r);
o.close();
String outputString = output.toString();
@ -401,7 +404,8 @@ public class TestOfflineImageViewer {
new Comparator<FileStatus>() {
@Override
public int compare(FileStatus first, FileStatus second) {
return first.getLen() < second.getLen() ? -1 :
return first.getLen() < second.getLen() ?
-1 :
((first.getLen() == second.getLen()) ? 0 : 1);
}
});
@ -410,6 +414,7 @@ public class TestOfflineImageViewer {
assertTrue(matcher.find() && matcher.groupCount() == 1);
assertEquals(maxFile.getLen(), Long.parseLong(matcher.group(1)));
}
}
@Test
public void testFileDistributionCalculatorWithOptions() throws Exception {
@ -514,7 +519,9 @@ public class TestOfflineImageViewer {
ByteArrayOutputStream output = new ByteArrayOutputStream();
PrintStream o = new PrintStream(output);
PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), o);
v.visit(new RandomAccessFile(originalFsimage, "r"));
try (RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) {
v.visit(r);
}
SAXParserFactory spf = SAXParserFactory.newInstance();
SAXParser parser = spf.newSAXParser();
final String xml = output.toString();
@ -667,10 +674,11 @@ public class TestOfflineImageViewer {
final String DELIMITER = "\t";
ByteArrayOutputStream output = new ByteArrayOutputStream();
try (PrintStream o = new PrintStream(output)) {
try (PrintStream o = new PrintStream(output);
RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) {
PBImageDelimitedTextWriter v =
new PBImageDelimitedTextWriter(o, DELIMITER, db);
v.visit(new RandomAccessFile(originalFsimage, "r"));
v.visit(r);
}
Set<String> fileNames = new HashSet<>();