Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-1920. Fixed TestFileSystemApplicationHistoryStore failure on windows. Contributed by Vinod Kumar Vavilapalli. 

svn merge --ignore-ancestry -c 1586414 ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1586420 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Zhijie Shen 2014-04-10 18:59:44 +00:00
parent 2dea526b23
commit 874b4979d0
3 changed files with 31 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -82,6 +82,9 @@ Release 2.4.1 - UNRELEASED
YARN-1910. Fixed a race condition in TestAMRMTokens that causes the test to YARN-1910. Fixed a race condition in TestAMRMTokens that causes the test to
fail more often on Windows. (Xuan Gong via vinodkv) fail more often on Windows. (Xuan Gong via vinodkv)
YARN-1920. Fixed TestFileSystemApplicationHistoryStore failure on windows.
(Vinod Kumar Vavilapalli via zjshen)
Release 2.4.0 - 2014-04-07 Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
View File

@ -179,7 +179,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService
LOG.info("Completed reading history information of application " + appId); LOG.info("Completed reading history information of application " + appId);
return historyData; return historyData;
} catch (IOException e) { } catch (IOException e) {
LOG.error("Error when reading history file of application " + appId); LOG.error("Error when reading history file of application " + appId, e);
throw e; throw e;
} finally { } finally {
hfReader.close(); hfReader.close();
@ -296,7 +296,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService
return historyData; return historyData;
} catch (IOException e) { } catch (IOException e) {
LOG.error("Error when reading history file of application attempt" LOG.error("Error when reading history file of application attempt"
+ appAttemptId); + appAttemptId, e);
throw e; throw e;
} finally { } finally {
hfReader.close(); hfReader.close();
@ -344,7 +344,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService
+ containerId); + containerId);
return historyData; return historyData;
} catch (IOException e) { } catch (IOException e) {
LOG.error("Error when reading history file of container " + containerId); LOG.error("Error when reading history file of container " + containerId, e);
throw e; throw e;
} finally { } finally {
hfReader.close(); hfReader.close();
@ -420,7 +420,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService
+ appStart.getApplicationId()); + appStart.getApplicationId());
} catch (IOException e) { } catch (IOException e) {
LOG.error("Error when openning history file of application " LOG.error("Error when openning history file of application "
+ appStart.getApplicationId()); + appStart.getApplicationId(), e);
throw e; throw e;
} }
outstandingWriters.put(appStart.getApplicationId(), hfWriter); outstandingWriters.put(appStart.getApplicationId(), hfWriter);
@ -437,7 +437,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService
+ appStart.getApplicationId() + " is written"); + appStart.getApplicationId() + " is written");
} catch (IOException e) { } catch (IOException e) {
LOG.error("Error when writing start information of application " LOG.error("Error when writing start information of application "
+ appStart.getApplicationId()); + appStart.getApplicationId(), e);
throw e; throw e;
} }
} }
@ -456,7 +456,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService
+ appFinish.getApplicationId() + " is written"); + appFinish.getApplicationId() + " is written");
} catch (IOException e) { } catch (IOException e) {
LOG.error("Error when writing finish information of application " LOG.error("Error when writing finish information of application "
+ appFinish.getApplicationId()); + appFinish.getApplicationId(), e);
throw e; throw e;
} finally { } finally {
hfWriter.close(); hfWriter.close();
@ -480,7 +480,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService
+ appAttemptStart.getApplicationAttemptId() + " is written"); + appAttemptStart.getApplicationAttemptId() + " is written");
} catch (IOException e) { } catch (IOException e) {
LOG.error("Error when writing start information of application attempt " LOG.error("Error when writing start information of application attempt "
+ appAttemptStart.getApplicationAttemptId()); + appAttemptStart.getApplicationAttemptId(), e);
throw e; throw e;
} }
} }
@ -501,7 +501,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService
+ appAttemptFinish.getApplicationAttemptId() + " is written"); + appAttemptFinish.getApplicationAttemptId() + " is written");
} catch (IOException e) { } catch (IOException e) {
LOG.error("Error when writing finish information of application attempt " LOG.error("Error when writing finish information of application attempt "
+ appAttemptFinish.getApplicationAttemptId()); + appAttemptFinish.getApplicationAttemptId(), e);
throw e; throw e;
} }
} }
@ -521,7 +521,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService
+ containerStart.getContainerId() + " is written"); + containerStart.getContainerId() + " is written");
} catch (IOException e) { } catch (IOException e) {
LOG.error("Error when writing start information of container " LOG.error("Error when writing start information of container "
+ containerStart.getContainerId()); + containerStart.getContainerId(), e);
throw e; throw e;
} }
} }
@ -541,7 +541,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService
+ containerFinish.getContainerId() + " is written"); + containerFinish.getContainerId() + " is written");
} catch (IOException e) { } catch (IOException e) {
LOG.error("Error when writing finish information of container " LOG.error("Error when writing finish information of container "
+ containerFinish.getContainerId()); + containerFinish.getContainerId(), e);
} }
} }
@ -676,9 +676,10 @@ public class FileSystemApplicationHistoryStore extends AbstractService
private TFile.Reader reader; private TFile.Reader reader;
private TFile.Reader.Scanner scanner; private TFile.Reader.Scanner scanner;
FSDataInputStream fsdis;
public HistoryFileReader(Path historyFile) throws IOException { public HistoryFileReader(Path historyFile) throws IOException {
FSDataInputStream fsdis = fs.open(historyFile); fsdis = fs.open(historyFile);
reader = reader =
new TFile.Reader(fsdis, fs.getFileStatus(historyFile).getLen(), new TFile.Reader(fsdis, fs.getFileStatus(historyFile).getLen(),
getConfig()); getConfig());
@ -707,7 +708,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService
} }
public void close() { public void close() {
IOUtils.cleanup(LOG, scanner, reader); IOUtils.cleanup(LOG, scanner, reader, fsdis);
} }
} }

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
View File

@ -23,6 +23,8 @@ import java.net.URI;
import org.junit.Assert; import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
@ -42,6 +44,9 @@ import org.junit.Test;
public class TestFileSystemApplicationHistoryStore extends public class TestFileSystemApplicationHistoryStore extends
ApplicationHistoryStoreTestUtils { ApplicationHistoryStoreTestUtils {
private static Log LOG = LogFactory
.getLog(TestFileSystemApplicationHistoryStore.class.getName());
private FileSystem fs; private FileSystem fs;
private Path fsWorkingPath; private Path fsWorkingPath;
@ -50,9 +55,12 @@ public class TestFileSystemApplicationHistoryStore extends
fs = new RawLocalFileSystem(); fs = new RawLocalFileSystem();
Configuration conf = new Configuration(); Configuration conf = new Configuration();
fs.initialize(new URI("/"), conf); fs.initialize(new URI("/"), conf);
fsWorkingPath = new Path("Test"); fsWorkingPath =
new Path("target",
TestFileSystemApplicationHistoryStore.class.getSimpleName());
fs.delete(fsWorkingPath, true); fs.delete(fsWorkingPath, true);
conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI, fsWorkingPath.toString()); conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI,
fsWorkingPath.toString());
store = new FileSystemApplicationHistoryStore(); store = new FileSystemApplicationHistoryStore();
store.init(conf); store.init(conf);
store.start(); store.start();
@ -67,6 +75,7 @@ public class TestFileSystemApplicationHistoryStore extends
@Test @Test
public void testReadWriteHistoryData() throws IOException { public void testReadWriteHistoryData() throws IOException {
LOG.info("Starting testReadWriteHistoryData");
testWriteHistoryData(5); testWriteHistoryData(5);
testReadHistoryData(5); testReadHistoryData(5);
} }
@ -167,6 +176,7 @@ public class TestFileSystemApplicationHistoryStore extends
@Test @Test
public void testWriteAfterApplicationFinish() throws IOException { public void testWriteAfterApplicationFinish() throws IOException {
LOG.info("Starting testWriteAfterApplicationFinish");
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
writeApplicationStartData(appId); writeApplicationStartData(appId);
writeApplicationFinishData(appId); writeApplicationFinishData(appId);
@ -203,6 +213,7 @@ public class TestFileSystemApplicationHistoryStore extends
@Test @Test
public void testMassiveWriteContainerHistoryData() throws IOException { public void testMassiveWriteContainerHistoryData() throws IOException {
LOG.info("Starting testMassiveWriteContainerHistoryData");
long mb = 1024 * 1024; long mb = 1024 * 1024;
long usedDiskBefore = fs.getContentSummary(fsWorkingPath).getLength() / mb; long usedDiskBefore = fs.getContentSummary(fsWorkingPath).getLength() / mb;
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
@ -221,12 +232,14 @@ public class TestFileSystemApplicationHistoryStore extends
@Test @Test
public void testMissingContainerHistoryData() throws IOException { public void testMissingContainerHistoryData() throws IOException {
LOG.info("Starting testMissingContainerHistoryData");
testWriteHistoryData(3, true, false); testWriteHistoryData(3, true, false);
testReadHistoryData(3, true, false); testReadHistoryData(3, true, false);
} }
@Test @Test
public void testMissingApplicationAttemptHistoryData() throws IOException { public void testMissingApplicationAttemptHistoryData() throws IOException {
LOG.info("Starting testMissingApplicationAttemptHistoryData");
testWriteHistoryData(3, false, true); testWriteHistoryData(3, false, true);
testReadHistoryData(3, false, true); testReadHistoryData(3, false, true);
} }