From d33e07dc49e00db138921fb3aa52c4ef00510161 Mon Sep 17 00:00:00 2001 From: Colin Patrick Mccabe Date: Wed, 29 Oct 2014 12:31:59 -0700 Subject: [PATCH] HDFS-7287. The OfflineImageViewer (OIV) can output invalid XML depending on the filename (Ravi Prakash via Colin P. McCabe) --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hdfs/tools/offlineImageViewer/PBImageXmlWriter.java | 3 ++- .../hdfs/tools/offlineImageViewer/XmlImageVisitor.java | 3 ++- .../tools/offlineImageViewer/TestOfflineImageViewer.java | 9 +++++++-- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index cae2b02796c..c0e3f73b82f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1113,6 +1113,9 @@ Release 2.6.0 - UNRELEASED HDFS-7128. Decommission slows way down when it gets towards the end. (Ming Ma via cnauroth) + HDFS-7287. The OfflineImageViewer (OIV) can output invalid XML depending on + the filename (Ravi Prakash via Colin P. McCabe) + BREAKDOWN OF HDFS-6584 ARCHIVAL STORAGE HDFS-6677. Change INodeFile and FSImage to support storage policy ID. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java index 99617b805ea..df00499626b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java @@ -49,6 +49,7 @@ import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotDiffSection; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotSection; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.StringTableSection; +import org.apache.hadoop.hdfs.util.XMLUtils; import org.apache.hadoop.io.IOUtils; import com.google.common.collect.Lists; @@ -410,7 +411,7 @@ public final class PBImageXmlWriter { } private PBImageXmlWriter o(final String e, final Object v) { - out.print("<" + e + ">" + v + ""); + out.print("<" + e + ">" + XMLUtils.mangleXmlString(v.toString()) + ""); return this; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/XmlImageVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/XmlImageVisitor.java index 939eb0c8654..2719109e9db 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/XmlImageVisitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/XmlImageVisitor.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.tools.offlineImageViewer; import java.io.IOException; import java.util.LinkedList; +import org.apache.hadoop.hdfs.util.XMLUtils; /** * An XmlImageVisitor walks over an fsimage structure and writes out * an equivalent XML document that contains the fsimage's components. @@ -83,6 +84,6 @@ public class XmlImageVisitor extends TextWriterImageVisitor { } private void writeTag(String tag, String value) throws IOException { - write("<" + tag + ">" + value + "\n"); + write("<" + tag + ">" + XMLUtils.mangleXmlString(value) + "\n"); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index 5c0d26c2ff5..36b5201bdc8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; import java.io.RandomAccessFile; @@ -125,6 +126,10 @@ public class TestOfflineImageViewer { hdfs.mkdirs(emptydir); writtenFiles.put(emptydir.toString(), hdfs.getFileStatus(emptydir)); + //Create a directory whose name should be escaped in XML + Path invalidXMLDir = new Path("/dirContainingInvalidXMLChar\u0000here"); + hdfs.mkdirs(invalidXMLDir); + // Get delegation tokens so we log the delegation token op Token[] delegationTokens = hdfs .addDelegationTokens(TEST_RENEWER, null); @@ -220,7 +225,7 @@ public class TestOfflineImageViewer { assertTrue(matcher.find() && matcher.groupCount() == 1); int totalDirs = Integer.parseInt(matcher.group(1)); // totalDirs includes root directory, empty directory, and xattr directory - assertEquals(NUM_DIRS + 3, totalDirs); + assertEquals(NUM_DIRS + 4, totalDirs); FileStatus maxFile = Collections.max(writtenFiles.values(), new Comparator() { @@ -272,7 +277,7 @@ public class TestOfflineImageViewer { // verify the number of directories FileStatus[] statuses = webhdfs.listStatus(new Path("/")); - assertEquals(NUM_DIRS + 2, statuses.length); // contains empty and xattr directory + assertEquals(NUM_DIRS + 3, statuses.length); // contains empty and xattr directory // verify the number of files in the directory statuses = webhdfs.listStatus(new Path("/dir0"));