Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-6604. The short-circuit cache doesn't correctly time out replicas that haven't been used in a while (cmccabe) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1607459 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Colin McCabe 2014-07-02 19:46:29 +00:00
parent c661b13e2e
commit a9cf4e7a98
3 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -498,6 +498,9 @@ Release 2.5.0 - UNRELEASED
HDFS-6591. while loop is executed tens of thousands of times in Hedged Read HDFS-6591. while loop is executed tens of thousands of times in Hedged Read
(Liang Xie via cnauroth) (Liang Xie via cnauroth)
HDFS-6604. The short-circuit cache doesn't correctly time out replicas that
haven't been used in a while (cmccabe)
BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS
HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh)

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java
View File

@ -111,7 +111,7 @@ public class ShortCircuitCache implements Closeable {
Long evictionTimeNs = Long.valueOf(0); Long evictionTimeNs = Long.valueOf(0);
while (true) { while (true) {
Entry<Long, ShortCircuitReplica> entry = Entry<Long, ShortCircuitReplica> entry =
evictableMmapped.ceilingEntry(evictionTimeNs); evictable.ceilingEntry(evictionTimeNs);
if (entry == null) break; if (entry == null) break;
evictionTimeNs = entry.getKey(); evictionTimeNs = entry.getKey();
long evictionTimeMs = long evictionTimeMs =
@ -384,10 +384,6 @@ public class ShortCircuitCache implements Closeable {
this.shmManager = shmManager; this.shmManager = shmManager;
} }
public long getMmapRetryTimeoutMs() {
return mmapRetryTimeoutMs;
}
public long getStaleThresholdMs() { public long getStaleThresholdMs() {
return staleThresholdMs; return staleThresholdMs;
} }
@ -847,7 +843,7 @@ public class ShortCircuitCache implements Closeable {
} else if (replica.mmapData instanceof Long) { } else if (replica.mmapData instanceof Long) {
long lastAttemptTimeMs = (Long)replica.mmapData; long lastAttemptTimeMs = (Long)replica.mmapData;
long delta = Time.monotonicNow() - lastAttemptTimeMs; long delta = Time.monotonicNow() - lastAttemptTimeMs;
if (delta < staleThresholdMs) { if (delta < mmapRetryTimeoutMs) {
if (LOG.isTraceEnabled()) { if (LOG.isTraceEnabled()) {
LOG.trace(this + ": can't create client mmap for " + LOG.trace(this + ": can't create client mmap for " +
replica + " because we failed to " + replica + " because we failed to " +

5
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
View File

@ -197,11 +197,12 @@ public class TestShortCircuitCache {
@Test(timeout=60000) @Test(timeout=60000)
public void testExpiry() throws Exception { public void testExpiry() throws Exception {
final ShortCircuitCache cache = final ShortCircuitCache cache =
new ShortCircuitCache(2, 1, 1, 10000000, 1, 10000, 0); new ShortCircuitCache(2, 1, 1, 10000000, 1, 10000000, 0);
final TestFileDescriptorPair pair = new TestFileDescriptorPair(); final TestFileDescriptorPair pair = new TestFileDescriptorPair();
ShortCircuitReplicaInfo replicaInfo1 = ShortCircuitReplicaInfo replicaInfo1 =
cache.fetchOrCreate( cache.fetchOrCreate(
new ExtendedBlockId(123, "test_bp1"), new SimpleReplicaCreator(123, cache, pair)); new ExtendedBlockId(123, "test_bp1"),
new SimpleReplicaCreator(123, cache, pair));
Preconditions.checkNotNull(replicaInfo1.getReplica()); Preconditions.checkNotNull(replicaInfo1.getReplica());
Preconditions.checkState(replicaInfo1.getInvalidTokenException() == null); Preconditions.checkState(replicaInfo1.getInvalidTokenException() == null);
pair.compareWith(replicaInfo1.getReplica().getDataStream(), pair.compareWith(replicaInfo1.getReplica().getDataStream(),