From 6efffc9cd8c45d7d7133c41598581c67ba424e50 Mon Sep 17 00:00:00 2001 From: Tsuyoshi Ozawa Date: Fri, 30 Jan 2015 04:42:05 +0900 Subject: [PATCH] YARN-3029. FSDownload.unpack() uses local locale for FS case conversion, may not work everywhere. Contributed by Varun Saxena. (cherry picked from commit 7acce7d3648d6f1e45ce280e2147e7dedf5693fc) --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../apache/hadoop/yarn/util/FSDownload.java | 5 +-- .../hadoop/yarn/util/TestFSDownload.java | 33 ++++++++++++++++--- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 847df29756c..ed6642cfa97 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -403,6 +403,9 @@ Release 2.7.0 - UNRELEASED YARN-3079. Scheduler should also update maximumAllocation when updateNodeResource. (Zhihai Xu via wangda) + YARN-3029. FSDownload.unpack() uses local locale for FS case conversion, may not + work everywhere. (Varun Saxena via ozawa) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java index 436cb312938..870aa952122 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java @@ -23,6 +23,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.net.URISyntaxException; import java.security.PrivilegedExceptionAction; +import java.util.Locale; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -271,7 +272,7 @@ public class FSDownload implements Callable { private long unpack(File localrsrc, File dst) throws IOException { switch (resource.getType()) { case ARCHIVE: { - String lowerDst = dst.getName().toLowerCase(); + String lowerDst = dst.getName().toLowerCase(Locale.ENGLISH); if (lowerDst.endsWith(".jar")) { RunJar.unJar(localrsrc, dst); } else if (lowerDst.endsWith(".zip")) { @@ -290,7 +291,7 @@ public class FSDownload implements Callable { } break; case PATTERN: { - String lowerDst = dst.getName().toLowerCase(); + String lowerDst = dst.getName().toLowerCase(Locale.ENGLISH); if (lowerDst.endsWith(".jar")) { String p = resource.getPattern(); RunJar.unJar(localrsrc, dst, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java index 07dd63006fb..3597b314715 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java @@ -33,6 +33,7 @@ import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Random; import java.util.concurrent.Callable; @@ -67,7 +68,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.util.Shell; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; @@ -232,7 +232,7 @@ public class TestFSDownload { byte[] bytes = new byte[len]; r.nextBytes(bytes); - File archiveFile = new File(p.toUri().getPath() + ".zip"); + File archiveFile = new File(p.toUri().getPath() + ".ZIP"); archiveFile.createNewFile(); ZipOutputStream out = new ZipOutputStream( new FileOutputStream(archiveFile)); @@ -243,11 +243,11 @@ public class TestFSDownload { LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString() - + ".zip"))); + + ".ZIP"))); ret.setSize(len); ret.setType(LocalResourceType.ARCHIVE); ret.setVisibility(vis); - ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".zip")) + ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".ZIP")) .getModificationTime()); return ret; } @@ -474,8 +474,8 @@ public class TestFSDownload { int size = rand.nextInt(512) + 512; LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE; - Path p = new Path(basedir, "" + 1); + String strFileName = ""; LocalResource rsrc = null; switch (fileType) { case TAR: @@ -487,6 +487,7 @@ public class TestFSDownload { break; case ZIP: rsrc = createZipFile(files, p, size, rand, vis); + strFileName = p.getName() + ".ZIP"; break; case TGZ: rsrc = createTgzFile(files, p, size, rand, vis); @@ -509,6 +510,13 @@ public class TestFSDownload { FileStatus[] childFiles = files.getDefaultFileSystem().listStatus( filestatus.getPath()); for (FileStatus childfile : childFiles) { + if(strFileName.endsWith(".ZIP") && + childfile.getPath().getName().equals(strFileName) && + !childfile.isDirectory()) { + Assert.fail("Failure...After unzip, there should have been a" + + " directory formed with zip file name but found a file. " + + childfile.getPath()); + } if (childfile.getPath().getName().startsWith("tmp")) { Assert.fail("Tmp File should not have been there " + childfile.getPath()); @@ -539,6 +547,21 @@ public class TestFSDownload { downloadWithFileType(TEST_FILE_TYPE.ZIP); } + /* + * To test fix for YARN-3029 + */ + @Test (timeout=10000) + public void testDownloadArchiveZipWithTurkishLocale() throws IOException, + URISyntaxException, InterruptedException { + Locale defaultLocale = Locale.getDefault(); + // Set to Turkish + Locale turkishLocale = new Locale("tr", "TR"); + Locale.setDefault(turkishLocale); + downloadWithFileType(TEST_FILE_TYPE.ZIP); + // Set the locale back to original default locale + Locale.setDefault(defaultLocale); + } + @Test (timeout=10000) public void testDownloadArchiveTgz() throws IOException, URISyntaxException, InterruptedException {