Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-11420. Edit file should not be processed by the same type processor in OfflineEditsViewer. Contributed by Yiqun Lin.

This commit is contained in:
Yiqun Lin 2017-03-15 18:05:03 +08:00
parent 2d0e24eb01
commit 4416a07c9c
3 changed files with 37 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java
View File

@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsLoader.OfflineEditsLoaderFactory;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@ -55,7 +56,9 @@ public class OfflineEditsViewer extends Configured implements Tool {
"Required command line arguments:\n" +
"-i,--inputFile <arg> edits file to process, xml (case\n" +
" insensitive) extension means XML format,\n" +
" any other filename means binary format\n" +
" any other filename means binary format.\n" +
" XML/Binary format input file is not allowed\n" +
" to be processed by the same type processor.\n" +
"-o,--outputFile <arg> Name of output file. If the specified\n" +
" file exists, it will be overwritten,\n" +
" format of the file is determined\n" +
@ -132,12 +135,24 @@ public class OfflineEditsViewer extends Configured implements Tool {
System.out.println("input [" + inputFileName + "]");
System.out.println("output [" + outputFileName + "]");
}
boolean xmlInput = StringUtils.toLowerCase(inputFileName).endsWith(".xml");
if (xmlInput && StringUtils.equalsIgnoreCase("xml", processor)) {
System.err.println("XML format input file is not allowed"
+ " to be processed by XML processor.");
return -1;
} else if(!xmlInput && StringUtils.equalsIgnoreCase("binary", processor)) {
System.err.println("Binary format input file is not allowed"
+ " to be processed by Binary processor.");
return -1;
}
try {
if (visitor == null) {
visitor = OfflineEditsVisitorFactory.getEditsVisitor(
outputFileName, processor, flags.getPrintToScreen());
}
boolean xmlInput = inputFileName.toLowerCase().endsWith(".xml");
OfflineEditsLoader loader = OfflineEditsLoaderFactory.
createLoader(visitor, inputFileName, xmlInput, flags);
loader.loadEdits();

2
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md
View File

@ -30,6 +30,8 @@ Input formats supported:
2. **xml**: XML format, as produced by xml processor, used if filename
has `.xml` (case insensitive) extension
Note: XML/Binary format input file is not allowed to be processed by the same type processor.
The Offline Edits Viewer provides several output processors (unless stated otherwise the output of the processor can be converted back to original edits file):
1. **binary**: native binary format that Hadoop uses internally

18
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
View File

@ -338,4 +338,22 @@ public class TestOfflineEditsViewer {
}
}
}
@Test
public void testProcessorWithSameTypeFormatFile() throws IOException {
String edits = nnHelper.generateEdits();
LOG.info("Generated edits=" + edits);
String binaryEdits = folder.newFile("binaryEdits").getAbsolutePath();
String editsParsedXml = folder.newFile("editsParsed.xml").getAbsolutePath();
String editsReparsedXml = folder.newFile("editsReparsed.xml")
.getAbsolutePath();
// Binary format input file is not allowed to be processed
// by Binary processor.
assertEquals(-1, runOev(edits, binaryEdits, "binary", false));
// parse to XML then back to XML
assertEquals(0, runOev(edits, editsParsedXml, "xml", false));
// XML format input file is not allowed to be processed by XML processor.
assertEquals(-1, runOev(editsParsedXml, editsReparsedXml, "xml", false));
}
}