From 3e5e5b028ad7e199d08e524fe7cddeee5db51a6d Mon Sep 17 00:00:00 2001 From: Srinivasu Majeti Date: Fri, 17 May 2019 19:19:12 +0200 Subject: [PATCH] HDFS-14323. Distcp fails in Hadoop 3.x when 2.x source webhdfs url has special characters in hdfs file path. Contributed by Srinivasu Majeti. Signed-off-by: Wei-Chiu Chuang --- .../org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index f9a64b340eb..fe30a9ac88f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -609,7 +609,13 @@ public class WebHdfsFileSystem extends FileSystem boolean pathAlreadyEncoded = false; try { fspathUriDecoded = URLDecoder.decode(fspathUri.getPath(), "UTF-8"); - pathAlreadyEncoded = true; + //below condition check added as part of fixing HDFS-14323 to make + //sure pathAlreadyEncoded is not set in the case the input url does + //not have any encoded sequence already.This will help pulling data + //from 2.x hadoop cluster to 3.x using 3.x distcp client operation + if(!fspathUri.getPath().equals(fspathUriDecoded)) { + pathAlreadyEncoded = true; + } } catch (IllegalArgumentException ex) { LOG.trace("Cannot decode URL encoded file", ex); }