honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Core: deletions (alone) should also force a refresh when version map is using too much RAM 

We already force a refresh in index/create ops, to clear version map
when it's using too much RAM, but we were failing to do this for
deletes, so an app that does tons of deletes with no indexing, and has
set refresh_interval to -1, would have version map using unbounded
RAM.

Closes #10312
This commit is contained in:
Michael McCandless 2015-03-30 10:25:45 -04:00 committed by mikemccand
parent 1200d8e647
commit 87aa39a0d2
2 changed files with 37 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/main/java/org/elasticsearch/index/engine/InternalEngine.java
View File

@ -447,6 +447,7 @@ public class InternalEngine extends Engine {
}
maybePruneDeletedTombstones();
checkVersionMapRefresh();
}
private void maybePruneDeletedTombstones() {

36
src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
View File

@ -52,6 +52,7 @@ import org.elasticsearch.index.codec.CodecService;
import org.elasticsearch.index.deletionpolicy.KeepOnlyLastDeletionPolicy;
import org.elasticsearch.index.deletionpolicy.SnapshotDeletionPolicy;
import org.elasticsearch.index.deletionpolicy.SnapshotIndexCommit;
import org.elasticsearch.index.engine.Engine.Searcher;
import org.elasticsearch.index.indexing.ShardIndexingService;
import org.elasticsearch.index.indexing.slowlog.ShardSlowLogIndexingService;
import org.elasticsearch.index.mapper.ParseContext.Document;
@ -1677,4 +1678,39 @@ public class InternalEngineTests extends ElasticsearchLuceneTestCase {
replicaSearcher.close();
}
// #10312
@Test
public void testDeletesAloneCanTriggerRefresh() throws IOException {
// Tiny indexing buffer:
Settings indexSettings = ImmutableSettings.builder().put(defaultSettings)
.put(EngineConfig.INDEX_BUFFER_SIZE_SETTING, "1kb").build();
IndexSettingsService indexSettingsService = new IndexSettingsService(shardId.index(), indexSettings);
try (Store store = createStore();
Translog translog = createTranslog();
Engine engine = new InternalEngine(config(indexSettingsService, store, translog, createMergeScheduler(indexSettingsService)))) {
for(int i=0;i<100;i++) {
String id = Integer.toString(i);
ParsedDocument doc = testParsedDocument(id, id, "test", null, -1, -1, testDocument(), B_1, false);
engine.index(new Engine.Index(null, newUid(id), doc, 2, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime()));
}
// Force merge so we know all merges are done before we start deleting:
engine.forceMerge(true, 1, false, false);
Searcher s = engine.acquireSearcher("test");
long version1 = ((DirectoryReader) s.reader()).getVersion();
s.close();
for(int i=0;i<100;i++) {
String id = Integer.toString(i);
engine.delete(new Engine.Delete("test", id, newUid(id), 10, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), false));
}
s = engine.acquireSearcher("test");
long version2 = ((DirectoryReader) s.reader()).getVersion();
s.close();
// 100 buffered deletes will easily exceed 25% of our 1 KB indexing buffer so it should have forced a refresh:
assertThat(version2, greaterThan(version1));
}
}
}