From 874b4979d00ef6c9c350efcb77bc452bcfea8754 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Thu, 10 Apr 2014 18:59:44 +0000 Subject: [PATCH] YARN-1920. Fixed TestFileSystemApplicationHistoryStore failure on windows. Contributed by Vinod Kumar Vavilapalli. svn merge --ignore-ancestry -c 1586414 ../../trunk/ git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1586420 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../FileSystemApplicationHistoryStore.java | 25 ++++++++++--------- ...TestFileSystemApplicationHistoryStore.java | 17 +++++++++++-- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 188a80035ac..e2af3191db8 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -82,6 +82,9 @@ Release 2.4.1 - UNRELEASED YARN-1910. Fixed a race condition in TestAMRMTokens that causes the test to fail more often on Windows. (Xuan Gong via vinodkv) + YARN-1920. Fixed TestFileSystemApplicationHistoryStore failure on windows. + (Vinod Kumar Vavilapalli via zjshen) + Release 2.4.0 - 2014-04-07 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java index 4c8d7452a25..a5725ebfffe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java @@ -179,7 +179,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService LOG.info("Completed reading history information of application " + appId); return historyData; } catch (IOException e) { - LOG.error("Error when reading history file of application " + appId); + LOG.error("Error when reading history file of application " + appId, e); throw e; } finally { hfReader.close(); @@ -296,7 +296,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService return historyData; } catch (IOException e) { LOG.error("Error when reading history file of application attempt" - + appAttemptId); + + appAttemptId, e); throw e; } finally { hfReader.close(); @@ -344,7 +344,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService + containerId); return historyData; } catch (IOException e) { - LOG.error("Error when reading history file of container " + containerId); + LOG.error("Error when reading history file of container " + containerId, e); throw e; } finally { hfReader.close(); @@ -420,7 +420,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService + appStart.getApplicationId()); } catch (IOException e) { LOG.error("Error when openning history file of application " - + appStart.getApplicationId()); + + appStart.getApplicationId(), e); throw e; } outstandingWriters.put(appStart.getApplicationId(), hfWriter); @@ -437,7 +437,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService + appStart.getApplicationId() + " is written"); } catch (IOException e) { LOG.error("Error when writing start information of application " - + appStart.getApplicationId()); + + appStart.getApplicationId(), e); throw e; } } @@ -456,7 +456,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService + appFinish.getApplicationId() + " is written"); } catch (IOException e) { LOG.error("Error when writing finish information of application " - + appFinish.getApplicationId()); + + appFinish.getApplicationId(), e); throw e; } finally { hfWriter.close(); @@ -480,7 +480,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService + appAttemptStart.getApplicationAttemptId() + " is written"); } catch (IOException e) { LOG.error("Error when writing start information of application attempt " - + appAttemptStart.getApplicationAttemptId()); + + appAttemptStart.getApplicationAttemptId(), e); throw e; } } @@ -501,7 +501,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService + appAttemptFinish.getApplicationAttemptId() + " is written"); } catch (IOException e) { LOG.error("Error when writing finish information of application attempt " - + appAttemptFinish.getApplicationAttemptId()); + + appAttemptFinish.getApplicationAttemptId(), e); throw e; } } @@ -521,7 +521,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService + containerStart.getContainerId() + " is written"); } catch (IOException e) { LOG.error("Error when writing start information of container " - + containerStart.getContainerId()); + + containerStart.getContainerId(), e); throw e; } } @@ -541,7 +541,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService + containerFinish.getContainerId() + " is written"); } catch (IOException e) { LOG.error("Error when writing finish information of container " - + containerFinish.getContainerId()); + + containerFinish.getContainerId(), e); } } @@ -676,9 +676,10 @@ public class FileSystemApplicationHistoryStore extends AbstractService private TFile.Reader reader; private TFile.Reader.Scanner scanner; + FSDataInputStream fsdis; public HistoryFileReader(Path historyFile) throws IOException { - FSDataInputStream fsdis = fs.open(historyFile); + fsdis = fs.open(historyFile); reader = new TFile.Reader(fsdis, fs.getFileStatus(historyFile).getLen(), getConfig()); @@ -707,7 +708,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService } public void close() { - IOUtils.cleanup(LOG, scanner, reader); + IOUtils.cleanup(LOG, scanner, reader, fsdis); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java index 627b7b2dd65..d31018c1181 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java @@ -23,6 +23,8 @@ import java.net.URI; import org.junit.Assert; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -42,6 +44,9 @@ import org.junit.Test; public class TestFileSystemApplicationHistoryStore extends ApplicationHistoryStoreTestUtils { + private static Log LOG = LogFactory + .getLog(TestFileSystemApplicationHistoryStore.class.getName()); + private FileSystem fs; private Path fsWorkingPath; @@ -50,9 +55,12 @@ public class TestFileSystemApplicationHistoryStore extends fs = new RawLocalFileSystem(); Configuration conf = new Configuration(); fs.initialize(new URI("/"), conf); - fsWorkingPath = new Path("Test"); + fsWorkingPath = + new Path("target", + TestFileSystemApplicationHistoryStore.class.getSimpleName()); fs.delete(fsWorkingPath, true); - conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI, fsWorkingPath.toString()); + conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI, + fsWorkingPath.toString()); store = new FileSystemApplicationHistoryStore(); store.init(conf); store.start(); @@ -67,6 +75,7 @@ public class TestFileSystemApplicationHistoryStore extends @Test public void testReadWriteHistoryData() throws IOException { + LOG.info("Starting testReadWriteHistoryData"); testWriteHistoryData(5); testReadHistoryData(5); } @@ -167,6 +176,7 @@ public class TestFileSystemApplicationHistoryStore extends @Test public void testWriteAfterApplicationFinish() throws IOException { + LOG.info("Starting testWriteAfterApplicationFinish"); ApplicationId appId = ApplicationId.newInstance(0, 1); writeApplicationStartData(appId); writeApplicationFinishData(appId); @@ -203,6 +213,7 @@ public class TestFileSystemApplicationHistoryStore extends @Test public void testMassiveWriteContainerHistoryData() throws IOException { + LOG.info("Starting testMassiveWriteContainerHistoryData"); long mb = 1024 * 1024; long usedDiskBefore = fs.getContentSummary(fsWorkingPath).getLength() / mb; ApplicationId appId = ApplicationId.newInstance(0, 1); @@ -221,12 +232,14 @@ public class TestFileSystemApplicationHistoryStore extends @Test public void testMissingContainerHistoryData() throws IOException { + LOG.info("Starting testMissingContainerHistoryData"); testWriteHistoryData(3, true, false); testReadHistoryData(3, true, false); } @Test public void testMissingApplicationAttemptHistoryData() throws IOException { + LOG.info("Starting testMissingApplicationAttemptHistoryData"); testWriteHistoryData(3, false, true); testReadHistoryData(3, false, true); }