HDFS-11420. Edit file should not be processed by the same type processor in OfflineEditsViewer. Contributed by Yiqun Lin.

This commit is contained in:
Yiqun Lin 2017-03-15 18:05:03 +08:00
parent 2d0e24eb01
commit 4416a07c9c
3 changed files with 37 additions and 2 deletions

View File

@ -22,6 +22,7 @@
import org.apache.hadoop.conf.Configured; import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsLoader.OfflineEditsLoaderFactory; import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsLoader.OfflineEditsLoaderFactory;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.util.ToolRunner;
@ -55,7 +56,9 @@ private void printHelp() {
"Required command line arguments:\n" + "Required command line arguments:\n" +
"-i,--inputFile <arg> edits file to process, xml (case\n" + "-i,--inputFile <arg> edits file to process, xml (case\n" +
" insensitive) extension means XML format,\n" + " insensitive) extension means XML format,\n" +
" any other filename means binary format\n" + " any other filename means binary format.\n" +
" XML/Binary format input file is not allowed\n" +
" to be processed by the same type processor.\n" +
"-o,--outputFile <arg> Name of output file. If the specified\n" + "-o,--outputFile <arg> Name of output file. If the specified\n" +
" file exists, it will be overwritten,\n" + " file exists, it will be overwritten,\n" +
" format of the file is determined\n" + " format of the file is determined\n" +
@ -132,12 +135,24 @@ public int go(String inputFileName, String outputFileName, String processor,
System.out.println("input [" + inputFileName + "]"); System.out.println("input [" + inputFileName + "]");
System.out.println("output [" + outputFileName + "]"); System.out.println("output [" + outputFileName + "]");
} }
boolean xmlInput = StringUtils.toLowerCase(inputFileName).endsWith(".xml");
if (xmlInput && StringUtils.equalsIgnoreCase("xml", processor)) {
System.err.println("XML format input file is not allowed"
+ " to be processed by XML processor.");
return -1;
} else if(!xmlInput && StringUtils.equalsIgnoreCase("binary", processor)) {
System.err.println("Binary format input file is not allowed"
+ " to be processed by Binary processor.");
return -1;
}
try { try {
if (visitor == null) { if (visitor == null) {
visitor = OfflineEditsVisitorFactory.getEditsVisitor( visitor = OfflineEditsVisitorFactory.getEditsVisitor(
outputFileName, processor, flags.getPrintToScreen()); outputFileName, processor, flags.getPrintToScreen());
} }
boolean xmlInput = inputFileName.toLowerCase().endsWith(".xml");
OfflineEditsLoader loader = OfflineEditsLoaderFactory. OfflineEditsLoader loader = OfflineEditsLoaderFactory.
createLoader(visitor, inputFileName, xmlInput, flags); createLoader(visitor, inputFileName, xmlInput, flags);
loader.loadEdits(); loader.loadEdits();

View File

@ -30,6 +30,8 @@ Input formats supported:
2. **xml**: XML format, as produced by xml processor, used if filename 2. **xml**: XML format, as produced by xml processor, used if filename
has `.xml` (case insensitive) extension has `.xml` (case insensitive) extension
Note: XML/Binary format input file is not allowed to be processed by the same type processor.
The Offline Edits Viewer provides several output processors (unless stated otherwise the output of the processor can be converted back to original edits file): The Offline Edits Viewer provides several output processors (unless stated otherwise the output of the processor can be converted back to original edits file):
1. **binary**: native binary format that Hadoop uses internally 1. **binary**: native binary format that Hadoop uses internally

View File

@ -338,4 +338,22 @@ public void testStatisticsStrWithNullOpCodeCount() throws IOException {
} }
} }
} }
@Test
public void testProcessorWithSameTypeFormatFile() throws IOException {
String edits = nnHelper.generateEdits();
LOG.info("Generated edits=" + edits);
String binaryEdits = folder.newFile("binaryEdits").getAbsolutePath();
String editsParsedXml = folder.newFile("editsParsed.xml").getAbsolutePath();
String editsReparsedXml = folder.newFile("editsReparsed.xml")
.getAbsolutePath();
// Binary format input file is not allowed to be processed
// by Binary processor.
assertEquals(-1, runOev(edits, binaryEdits, "binary", false));
// parse to XML then back to XML
assertEquals(0, runOev(edits, editsParsedXml, "xml", false));
// XML format input file is not allowed to be processed by XML processor.
assertEquals(-1, runOev(editsParsedXml, editsReparsedXml, "xml", false));
}
} }