honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove cache key renderer argument from IndicesRequestCache (#62534) 

In the context of of a recurring test failure tracked by #32827, we added trace logging and an extra cache key renderer argument to IndicesRequestCache#getOrCompute (see #39475 and #34180).

We addressed the issue with #54071, but the extra argument was left behind, with a NORELEASE comment saying it should be removed.

With this commit, we remove the extra cache key rendered argument and the corresponding log lines which are not so useful without it.

Closes #55837
This commit is contained in:
Luca Cavanna 2020-09-19 00:24:02 +02:00 committed by GitHub
parent 8aeab0bec9
commit 00272ea877
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 23 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
server/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.indices;
import com.carrotsearch.hppc.ObjectHashSet; import com.carrotsearch.hppc.ObjectHashSet;
import com.carrotsearch.hppc.ObjectSet; import com.carrotsearch.hppc.ObjectSet;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DirectoryReader;
@ -51,7 +50,6 @@ import java.util.Iterator;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.function.Supplier;
/** /**
* The indices request cache allows to cache a shard level request stage responses, helping with improving * The indices request cache allows to cache a shard level request stage responses, helping with improving
@ -114,20 +112,14 @@ public final class IndicesRequestCache implements RemovalListener<IndicesRequest
notification.getKey().entity.onRemoval(notification); notification.getKey().entity.onRemoval(notification);
} }
// NORELEASE The cacheKeyRenderer has been added in order to debug
// https://github.com/elastic/elasticsearch/issues/32827, it should be
// removed when this issue is solved
BytesReference getOrCompute(CacheEntity cacheEntity, CheckedSupplier<BytesReference, IOException> loader, BytesReference getOrCompute(CacheEntity cacheEntity, CheckedSupplier<BytesReference, IOException> loader,
DirectoryReader reader, BytesReference cacheKey, Supplier<String> cacheKeyRenderer) throws Exception { DirectoryReader reader, BytesReference cacheKey) throws Exception {
assert reader.getReaderCacheHelper() != null; assert reader.getReaderCacheHelper() != null;
final Key key = new Key(cacheEntity, reader.getReaderCacheHelper().getKey(), cacheKey); final Key key = new Key(cacheEntity, reader.getReaderCacheHelper().getKey(), cacheKey);
Loader cacheLoader = new Loader(cacheEntity, loader); Loader cacheLoader = new Loader(cacheEntity, loader);
BytesReference value = cache.computeIfAbsent(key, cacheLoader); BytesReference value = cache.computeIfAbsent(key, cacheLoader);
if (cacheLoader.isLoaded()) { if (cacheLoader.isLoaded()) {
key.entity.onMiss(); key.entity.onMiss();
if (logger.isTraceEnabled()) {
logger.trace("Cache miss for reader version [{}] and request:\n {}", reader.getVersion(), cacheKeyRenderer.get());
}
// see if its the first time we see this reader, and make sure to register a cleanup key // see if its the first time we see this reader, and make sure to register a cleanup key
CleanupKey cleanupKey = new CleanupKey(cacheEntity, reader.getReaderCacheHelper().getKey()); CleanupKey cleanupKey = new CleanupKey(cacheEntity, reader.getReaderCacheHelper().getKey());
if (!registeredClosedListeners.containsKey(cleanupKey)) { if (!registeredClosedListeners.containsKey(cleanupKey)) {
@ -138,9 +130,6 @@ public final class IndicesRequestCache implements RemovalListener<IndicesRequest
} }
} else { } else {
key.entity.onHit(); key.entity.onHit();
if (logger.isTraceEnabled()) {
logger.trace("Cache hit for reader version [{}] and request:\n {}", reader.getVersion(), cacheKeyRenderer.get());
}
} }
return value; return value;
} }

6
server/src/main/java/org/elasticsearch/indices/IndicesService.java
View File

@ -161,7 +161,6 @@ import java.util.function.Consumer;
import java.util.function.Function; import java.util.function.Function;
import java.util.function.LongSupplier; import java.util.function.LongSupplier;
import java.util.function.Predicate; import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static java.util.Collections.emptyList; import static java.util.Collections.emptyList;
@ -1388,7 +1387,6 @@ public class IndicesService extends AbstractLifecycleComponent
boolean[] loadedFromCache = new boolean[] { true }; boolean[] loadedFromCache = new boolean[] { true };
BytesReference bytesReference = cacheShardLevelResult(context.indexShard(), directoryReader, request.cacheKey(), BytesReference bytesReference = cacheShardLevelResult(context.indexShard(), directoryReader, request.cacheKey(),
() -> "Shard: " + request.shardId() + "\nSource:\n" + request.source(),
out -> { out -> {
queryPhase.execute(context); queryPhase.execute(context);
context.queryResult().writeToNoId(out); context.queryResult().writeToNoId(out);
@ -1430,7 +1428,7 @@ public class IndicesService extends AbstractLifecycleComponent
* @return the contents of the cache or the result of calling the loader * @return the contents of the cache or the result of calling the loader
*/ */
private BytesReference cacheShardLevelResult(IndexShard shard, DirectoryReader reader, BytesReference cacheKey, private BytesReference cacheShardLevelResult(IndexShard shard, DirectoryReader reader, BytesReference cacheKey,
Supplier<String> cacheKeyRenderer, CheckedConsumer<StreamOutput, IOException> loader) throws Exception { CheckedConsumer<StreamOutput, IOException> loader) throws Exception {
IndexShardCacheEntity cacheEntity = new IndexShardCacheEntity(shard); IndexShardCacheEntity cacheEntity = new IndexShardCacheEntity(shard);
CheckedSupplier<BytesReference, IOException> supplier = () -> { CheckedSupplier<BytesReference, IOException> supplier = () -> {
/* BytesStreamOutput allows to pass the expected size but by default uses /* BytesStreamOutput allows to pass the expected size but by default uses
@ -1448,7 +1446,7 @@ public class IndicesService extends AbstractLifecycleComponent
return out.bytes(); return out.bytes();
} }
}; };
return indicesRequestCache.getOrCompute(cacheEntity, supplier, reader, cacheKey, cacheKeyRenderer); return indicesRequestCache.getOrCompute(cacheEntity, supplier, reader, cacheKey);
} }
static final class IndexShardCacheEntity extends AbstractIndexShardCacheEntity { static final class IndexShardCacheEntity extends AbstractIndexShardCacheEntity {

36
server/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
View File

@ -69,7 +69,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
// initial cache // initial cache
TestEntity entity = new TestEntity(requestCacheStats, indexShard); TestEntity entity = new TestEntity(requestCacheStats, indexShard);
Loader loader = new Loader(reader, 0); Loader loader = new Loader(reader, 0);
BytesReference value = cache.getOrCompute(entity, loader, reader, termBytes, () -> termQuery.toString()); BytesReference value = cache.getOrCompute(entity, loader, reader, termBytes);
assertEquals("foo", value.streamInput().readString()); assertEquals("foo", value.streamInput().readString());
assertEquals(0, requestCacheStats.stats().getHitCount()); assertEquals(0, requestCacheStats.stats().getHitCount());
assertEquals(1, requestCacheStats.stats().getMissCount()); assertEquals(1, requestCacheStats.stats().getMissCount());
@ -80,7 +80,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
// cache hit // cache hit
entity = new TestEntity(requestCacheStats, indexShard); entity = new TestEntity(requestCacheStats, indexShard);
loader = new Loader(reader, 0); loader = new Loader(reader, 0);
value = cache.getOrCompute(entity, loader, reader, termBytes, () -> termQuery.toString()); value = cache.getOrCompute(entity, loader, reader, termBytes);
assertEquals("foo", value.streamInput().readString()); assertEquals("foo", value.streamInput().readString());
assertEquals(1, requestCacheStats.stats().getHitCount()); assertEquals(1, requestCacheStats.stats().getHitCount());
assertEquals(1, requestCacheStats.stats().getMissCount()); assertEquals(1, requestCacheStats.stats().getMissCount());
@ -131,7 +131,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
// initial cache // initial cache
TestEntity entity = new TestEntity(requestCacheStats, indexShard); TestEntity entity = new TestEntity(requestCacheStats, indexShard);
Loader loader = new Loader(reader, 0); Loader loader = new Loader(reader, 0);
BytesReference value = cache.getOrCompute(entity, loader, reader, termBytes, () -> termQuery.toString()); BytesReference value = cache.getOrCompute(entity, loader, reader, termBytes);
assertEquals("foo", value.streamInput().readString()); assertEquals("foo", value.streamInput().readString());
assertEquals(0, requestCacheStats.stats().getHitCount()); assertEquals(0, requestCacheStats.stats().getHitCount());
assertEquals(1, requestCacheStats.stats().getMissCount()); assertEquals(1, requestCacheStats.stats().getMissCount());
@ -145,7 +145,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
// cache the second // cache the second
TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard); TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard);
loader = new Loader(secondReader, 0); loader = new Loader(secondReader, 0);
value = cache.getOrCompute(entity, loader, secondReader, termBytes, () -> termQuery.toString()); value = cache.getOrCompute(entity, loader, secondReader, termBytes);
assertEquals("bar", value.streamInput().readString()); assertEquals("bar", value.streamInput().readString());
assertEquals(0, requestCacheStats.stats().getHitCount()); assertEquals(0, requestCacheStats.stats().getHitCount());
assertEquals(2, requestCacheStats.stats().getMissCount()); assertEquals(2, requestCacheStats.stats().getMissCount());
@ -157,7 +157,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
secondEntity = new TestEntity(requestCacheStats, indexShard); secondEntity = new TestEntity(requestCacheStats, indexShard);
loader = new Loader(secondReader, 0); loader = new Loader(secondReader, 0);
value = cache.getOrCompute(secondEntity, loader, secondReader, termBytes, () -> termQuery.toString()); value = cache.getOrCompute(secondEntity, loader, secondReader, termBytes);
assertEquals("bar", value.streamInput().readString()); assertEquals("bar", value.streamInput().readString());
assertEquals(1, requestCacheStats.stats().getHitCount()); assertEquals(1, requestCacheStats.stats().getHitCount());
assertEquals(2, requestCacheStats.stats().getMissCount()); assertEquals(2, requestCacheStats.stats().getMissCount());
@ -167,7 +167,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
entity = new TestEntity(requestCacheStats, indexShard); entity = new TestEntity(requestCacheStats, indexShard);
loader = new Loader(reader, 0); loader = new Loader(reader, 0);
value = cache.getOrCompute(entity, loader, reader, termBytes, () -> termQuery.toString()); value = cache.getOrCompute(entity, loader, reader, termBytes);
assertEquals("foo", value.streamInput().readString()); assertEquals("foo", value.streamInput().readString());
assertEquals(2, requestCacheStats.stats().getHitCount()); assertEquals(2, requestCacheStats.stats().getHitCount());
assertEquals(2, requestCacheStats.stats().getMissCount()); assertEquals(2, requestCacheStats.stats().getMissCount());
@ -227,9 +227,9 @@ public class IndicesRequestCacheTests extends ESTestCase {
TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard); TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard);
Loader secondLoader = new Loader(secondReader, 0); Loader secondLoader = new Loader(secondReader, 0);
BytesReference value1 = cache.getOrCompute(entity, loader, reader, termBytes, () -> termQuery.toString()); BytesReference value1 = cache.getOrCompute(entity, loader, reader, termBytes);
assertEquals("foo", value1.streamInput().readString()); assertEquals("foo", value1.streamInput().readString());
BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termBytes, () -> termQuery.toString()); BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termBytes);
assertEquals("bar", value2.streamInput().readString()); assertEquals("bar", value2.streamInput().readString());
size = requestCacheStats.stats().getMemorySize(); size = requestCacheStats.stats().getMemorySize();
IOUtils.close(reader, secondReader, writer, dir, cache); IOUtils.close(reader, secondReader, writer, dir, cache);
@ -262,12 +262,12 @@ public class IndicesRequestCacheTests extends ESTestCase {
TestEntity thirddEntity = new TestEntity(requestCacheStats, indexShard); TestEntity thirddEntity = new TestEntity(requestCacheStats, indexShard);
Loader thirdLoader = new Loader(thirdReader, 0); Loader thirdLoader = new Loader(thirdReader, 0);
BytesReference value1 = cache.getOrCompute(entity, loader, reader, termBytes, () -> termQuery.toString()); BytesReference value1 = cache.getOrCompute(entity, loader, reader, termBytes);
assertEquals("foo", value1.streamInput().readString()); assertEquals("foo", value1.streamInput().readString());
BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termBytes, () -> termQuery.toString()); BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termBytes);
assertEquals("bar", value2.streamInput().readString()); assertEquals("bar", value2.streamInput().readString());
logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize()); logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize());
BytesReference value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termBytes, () -> termQuery.toString()); BytesReference value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termBytes);
assertEquals("baz", value3.streamInput().readString()); assertEquals("baz", value3.streamInput().readString());
assertEquals(2, cache.count()); assertEquals(2, cache.count());
assertEquals(1, requestCacheStats.stats().getEvictions()); assertEquals(1, requestCacheStats.stats().getEvictions());
@ -303,12 +303,12 @@ public class IndicesRequestCacheTests extends ESTestCase {
TestEntity thirddEntity = new TestEntity(requestCacheStats, differentIdentity); TestEntity thirddEntity = new TestEntity(requestCacheStats, differentIdentity);
Loader thirdLoader = new Loader(thirdReader, 0); Loader thirdLoader = new Loader(thirdReader, 0);
BytesReference value1 = cache.getOrCompute(entity, loader, reader, termBytes, () -> termQuery.toString()); BytesReference value1 = cache.getOrCompute(entity, loader, reader, termBytes);
assertEquals("foo", value1.streamInput().readString()); assertEquals("foo", value1.streamInput().readString());
BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termBytes, () -> termQuery.toString()); BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termBytes);
assertEquals("bar", value2.streamInput().readString()); assertEquals("bar", value2.streamInput().readString());
logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize()); logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize());
BytesReference value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termBytes, () -> termQuery.toString()); BytesReference value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termBytes);
assertEquals("baz", value3.streamInput().readString()); assertEquals("baz", value3.streamInput().readString());
assertEquals(3, cache.count()); assertEquals(3, cache.count());
final long hitCount = requestCacheStats.stats().getHitCount(); final long hitCount = requestCacheStats.stats().getHitCount();
@ -317,7 +317,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
cache.cleanCache(); cache.cleanCache();
assertEquals(1, cache.count()); assertEquals(1, cache.count());
// third has not been validated since it's a different identity // third has not been validated since it's a different identity
value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termBytes, () -> termQuery.toString()); value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termBytes);
assertEquals(hitCount + 1, requestCacheStats.stats().getHitCount()); assertEquals(hitCount + 1, requestCacheStats.stats().getHitCount());
assertEquals("baz", value3.streamInput().readString()); assertEquals("baz", value3.streamInput().readString());
@ -376,7 +376,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
// initial cache // initial cache
TestEntity entity = new TestEntity(requestCacheStats, indexShard); TestEntity entity = new TestEntity(requestCacheStats, indexShard);
Loader loader = new Loader(reader, 0); Loader loader = new Loader(reader, 0);
BytesReference value = cache.getOrCompute(entity, loader, reader, termBytes, () -> termQuery.toString()); BytesReference value = cache.getOrCompute(entity, loader, reader, termBytes);
assertEquals("foo", value.streamInput().readString()); assertEquals("foo", value.streamInput().readString());
assertEquals(0, requestCacheStats.stats().getHitCount()); assertEquals(0, requestCacheStats.stats().getHitCount());
assertEquals(1, requestCacheStats.stats().getMissCount()); assertEquals(1, requestCacheStats.stats().getMissCount());
@ -387,7 +387,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
// cache hit // cache hit
entity = new TestEntity(requestCacheStats, indexShard); entity = new TestEntity(requestCacheStats, indexShard);
loader = new Loader(reader, 0); loader = new Loader(reader, 0);
value = cache.getOrCompute(entity, loader, reader, termBytes, () -> termQuery.toString()); value = cache.getOrCompute(entity, loader, reader, termBytes);
assertEquals("foo", value.streamInput().readString()); assertEquals("foo", value.streamInput().readString());
assertEquals(1, requestCacheStats.stats().getHitCount()); assertEquals(1, requestCacheStats.stats().getHitCount());
assertEquals(1, requestCacheStats.stats().getMissCount()); assertEquals(1, requestCacheStats.stats().getMissCount());
@ -401,7 +401,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
entity = new TestEntity(requestCacheStats, indexShard); entity = new TestEntity(requestCacheStats, indexShard);
loader = new Loader(reader, 0); loader = new Loader(reader, 0);
cache.invalidate(entity, reader, termBytes); cache.invalidate(entity, reader, termBytes);
value = cache.getOrCompute(entity, loader, reader, termBytes, () -> termQuery.toString()); value = cache.getOrCompute(entity, loader, reader, termBytes);
assertEquals("foo", value.streamInput().readString()); assertEquals("foo", value.streamInput().readString());
assertEquals(1, requestCacheStats.stats().getHitCount()); assertEquals(1, requestCacheStats.stats().getHitCount());
assertEquals(2, requestCacheStats.stats().getMissCount()); assertEquals(2, requestCacheStats.stats().getMissCount());

4
server/src/test/java/org/elasticsearch/indices/IndicesServiceCloseTests.java
View File

@ -48,8 +48,8 @@ import org.elasticsearch.transport.nio.MockNioTransportPlugin;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.Arrays; import java.util.Arrays;
import java.util.concurrent.TimeUnit;
import java.util.Collections; import java.util.Collections;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING; import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING;
import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
@ -287,7 +287,7 @@ public class IndicesServiceCloseTests extends ESTestCase {
@Override @Override
public void onRemoval(RemovalNotification<Key, BytesReference> notification) {} public void onRemoval(RemovalNotification<Key, BytesReference> notification) {}
}; };
cache.getOrCompute(cacheEntity, () -> new BytesArray("bar"), searcher.getDirectoryReader(), new BytesArray("foo"), () -> "foo"); cache.getOrCompute(cacheEntity, () -> new BytesArray("bar"), searcher.getDirectoryReader(), new BytesArray("foo"));
assertEquals(1L, cache.count()); assertEquals(1L, cache.count());
searcher.close(); searcher.close();