From 4416a07c9c57af01ffbd2e43afcde2a1ba94c736 Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Wed, 15 Mar 2017 18:05:03 +0800 Subject: [PATCH] HDFS-11420. Edit file should not be processed by the same type processor in OfflineEditsViewer. Contributed by Yiqun Lin. --- .../OfflineEditsViewer.java | 19 +++++++++++++++++-- .../src/site/markdown/HdfsEditsViewer.md | 2 ++ .../TestOfflineEditsViewer.java | 18 ++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java index 107881ff590..f075ed29ec4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java @@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsLoader.OfflineEditsLoaderFactory; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; @@ -55,7 +56,9 @@ public class OfflineEditsViewer extends Configured implements Tool { "Required command line arguments:\n" + "-i,--inputFile edits file to process, xml (case\n" + " insensitive) extension means XML format,\n" + - " any other filename means binary format\n" + + " any other filename means binary format.\n" + + " XML/Binary format input file is not allowed\n" + + " to be processed by the same type processor.\n" + "-o,--outputFile Name of output file. If the specified\n" + " file exists, it will be overwritten,\n" + " format of the file is determined\n" + @@ -132,12 +135,24 @@ public class OfflineEditsViewer extends Configured implements Tool { System.out.println("input [" + inputFileName + "]"); System.out.println("output [" + outputFileName + "]"); } + + boolean xmlInput = StringUtils.toLowerCase(inputFileName).endsWith(".xml"); + if (xmlInput && StringUtils.equalsIgnoreCase("xml", processor)) { + System.err.println("XML format input file is not allowed" + + " to be processed by XML processor."); + return -1; + } else if(!xmlInput && StringUtils.equalsIgnoreCase("binary", processor)) { + System.err.println("Binary format input file is not allowed" + + " to be processed by Binary processor."); + return -1; + } + try { if (visitor == null) { visitor = OfflineEditsVisitorFactory.getEditsVisitor( outputFileName, processor, flags.getPrintToScreen()); } - boolean xmlInput = inputFileName.toLowerCase().endsWith(".xml"); + OfflineEditsLoader loader = OfflineEditsLoaderFactory. createLoader(visitor, inputFileName, xmlInput, flags); loader.loadEdits(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md index 5e069bbacea..4ab07ce2143 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md @@ -30,6 +30,8 @@ Input formats supported: 2. **xml**: XML format, as produced by xml processor, used if filename has `.xml` (case insensitive) extension +Note: XML/Binary format input file is not allowed to be processed by the same type processor. + The Offline Edits Viewer provides several output processors (unless stated otherwise the output of the processor can be converted back to original edits file): 1. **binary**: native binary format that Hadoop uses internally diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java index a21bc8f1299..bbad73c0418 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java @@ -338,4 +338,22 @@ public class TestOfflineEditsViewer { } } } + + @Test + public void testProcessorWithSameTypeFormatFile() throws IOException { + String edits = nnHelper.generateEdits(); + LOG.info("Generated edits=" + edits); + String binaryEdits = folder.newFile("binaryEdits").getAbsolutePath(); + String editsParsedXml = folder.newFile("editsParsed.xml").getAbsolutePath(); + String editsReparsedXml = folder.newFile("editsReparsed.xml") + .getAbsolutePath(); + + // Binary format input file is not allowed to be processed + // by Binary processor. + assertEquals(-1, runOev(edits, binaryEdits, "binary", false)); + // parse to XML then back to XML + assertEquals(0, runOev(edits, editsParsedXml, "xml", false)); + // XML format input file is not allowed to be processed by XML processor. + assertEquals(-1, runOev(editsParsedXml, editsReparsedXml, "xml", false)); + } }