HDFS-11420. Edit file should not be processed by the same type processor in OfflineEditsViewer. Contributed by Yiqun Lin.
This commit is contained in:
parent
caaa6fa96b
commit
c0e061b410
|
@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configured;
|
import org.apache.hadoop.conf.Configured;
|
||||||
import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsLoader.OfflineEditsLoaderFactory;
|
import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsLoader.OfflineEditsLoaderFactory;
|
||||||
|
import org.apache.hadoop.util.StringUtils;
|
||||||
import org.apache.hadoop.util.Tool;
|
import org.apache.hadoop.util.Tool;
|
||||||
import org.apache.hadoop.util.ToolRunner;
|
import org.apache.hadoop.util.ToolRunner;
|
||||||
|
|
||||||
|
@ -55,7 +56,9 @@ public class OfflineEditsViewer extends Configured implements Tool {
|
||||||
"Required command line arguments:\n" +
|
"Required command line arguments:\n" +
|
||||||
"-i,--inputFile <arg> edits file to process, xml (case\n" +
|
"-i,--inputFile <arg> edits file to process, xml (case\n" +
|
||||||
" insensitive) extension means XML format,\n" +
|
" insensitive) extension means XML format,\n" +
|
||||||
" any other filename means binary format\n" +
|
" any other filename means binary format.\n" +
|
||||||
|
" XML/Binary format input file is not allowed\n" +
|
||||||
|
" to be processed by the same type processor.\n" +
|
||||||
"-o,--outputFile <arg> Name of output file. If the specified\n" +
|
"-o,--outputFile <arg> Name of output file. If the specified\n" +
|
||||||
" file exists, it will be overwritten,\n" +
|
" file exists, it will be overwritten,\n" +
|
||||||
" format of the file is determined\n" +
|
" format of the file is determined\n" +
|
||||||
|
@ -132,12 +135,24 @@ public class OfflineEditsViewer extends Configured implements Tool {
|
||||||
System.out.println("input [" + inputFileName + "]");
|
System.out.println("input [" + inputFileName + "]");
|
||||||
System.out.println("output [" + outputFileName + "]");
|
System.out.println("output [" + outputFileName + "]");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
boolean xmlInput = StringUtils.toLowerCase(inputFileName).endsWith(".xml");
|
||||||
|
if (xmlInput && StringUtils.equalsIgnoreCase("xml", processor)) {
|
||||||
|
System.err.println("XML format input file is not allowed"
|
||||||
|
+ " to be processed by XML processor.");
|
||||||
|
return -1;
|
||||||
|
} else if(!xmlInput && StringUtils.equalsIgnoreCase("binary", processor)) {
|
||||||
|
System.err.println("Binary format input file is not allowed"
|
||||||
|
+ " to be processed by Binary processor.");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (visitor == null) {
|
if (visitor == null) {
|
||||||
visitor = OfflineEditsVisitorFactory.getEditsVisitor(
|
visitor = OfflineEditsVisitorFactory.getEditsVisitor(
|
||||||
outputFileName, processor, flags.getPrintToScreen());
|
outputFileName, processor, flags.getPrintToScreen());
|
||||||
}
|
}
|
||||||
boolean xmlInput = inputFileName.toLowerCase().endsWith(".xml");
|
|
||||||
OfflineEditsLoader loader = OfflineEditsLoaderFactory.
|
OfflineEditsLoader loader = OfflineEditsLoaderFactory.
|
||||||
createLoader(visitor, inputFileName, xmlInput, flags);
|
createLoader(visitor, inputFileName, xmlInput, flags);
|
||||||
loader.loadEdits();
|
loader.loadEdits();
|
||||||
|
|
|
@ -30,6 +30,8 @@ Input formats supported:
|
||||||
2. **xml**: XML format, as produced by xml processor, used if filename
|
2. **xml**: XML format, as produced by xml processor, used if filename
|
||||||
has `.xml` (case insensitive) extension
|
has `.xml` (case insensitive) extension
|
||||||
|
|
||||||
|
Note: XML/Binary format input file is not allowed to be processed by the same type processor.
|
||||||
|
|
||||||
The Offline Edits Viewer provides several output processors (unless stated otherwise the output of the processor can be converted back to original edits file):
|
The Offline Edits Viewer provides several output processors (unless stated otherwise the output of the processor can be converted back to original edits file):
|
||||||
|
|
||||||
1. **binary**: native binary format that Hadoop uses internally
|
1. **binary**: native binary format that Hadoop uses internally
|
||||||
|
|
|
@ -308,4 +308,22 @@ public class TestOfflineEditsViewer {
|
||||||
IOUtils.closeStream(out);
|
IOUtils.closeStream(out);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testProcessorWithSameTypeFormatFile() throws IOException {
|
||||||
|
String edits = nnHelper.generateEdits();
|
||||||
|
LOG.info("Generated edits=" + edits);
|
||||||
|
String binaryEdits = folder.newFile("binaryEdits").getAbsolutePath();
|
||||||
|
String editsParsedXml = folder.newFile("editsParsed.xml").getAbsolutePath();
|
||||||
|
String editsReparsedXml = folder.newFile("editsReparsed.xml")
|
||||||
|
.getAbsolutePath();
|
||||||
|
|
||||||
|
// Binary format input file is not allowed to be processed
|
||||||
|
// by Binary processor.
|
||||||
|
assertEquals(-1, runOev(edits, binaryEdits, "binary", false));
|
||||||
|
// parse to XML then back to XML
|
||||||
|
assertEquals(0, runOev(edits, editsParsedXml, "xml", false));
|
||||||
|
// XML format input file is not allowed to be processed by XML processor.
|
||||||
|
assertEquals(-1, runOev(editsParsedXml, editsReparsedXml, "xml", false));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue