From 2dc4af2b9379e5257dda4ea5fe551dc8e4af5226 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Thu, 5 Feb 2015 16:40:09 +0000 Subject: [PATCH] MAPREDUCE-6059. Speed up history server startup time (Siqi Li via aw) (cherry picked from commit fd57ab2002f97dcc83d455a5e0c770c8efde77a4) Conflicts: hadoop-mapreduce-project/CHANGES.txt --- hadoop-mapreduce-project/CHANGES.txt | 2 ++ .../apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 12fd8a09f7b..7dfa4543a52 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -48,6 +48,8 @@ Release 2.7.0 - UNRELEASED from key and value at the end of the iteration to save memory. (Zhihai Xu via kasha) + MAPREDUCE-6059. Speed up history server startup time (Siqi Li via aw) + BUG FIXES MAPREDUCE-6210. Use getApplicationAttemptId() instead of getApplicationId() diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index c2ccdb8d190..6b9f1462a6f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -263,6 +263,10 @@ public class HistoryFileManager extends AbstractService { public HistoryFileInfo get(JobId jobId) { return cache.get(jobId); } + + public boolean isFull() { + return cache.size() >= maxSize; + } } /** @@ -668,6 +672,10 @@ public class HistoryFileManager extends AbstractService { for (FileStatus fs : timestampedDirList) { // TODO Could verify the correct format for these directories. addDirectoryToSerialNumberIndex(fs.getPath()); + } + for (int i= timestampedDirList.size() - 1; + i >= 0 && !jobListCache.isFull(); i--) { + FileStatus fs = timestampedDirList.get(i); addDirectoryToJobListCache(fs.getPath()); } }