From 2d77e2a37e278197973ea6469d0eb88bcb32e87a Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Mon, 23 Dec 2013 11:00:34 +0100 Subject: [PATCH] Disable SegmentReader ram usage by default even if -ea is provided --- .../index/engine/robin/RobinEngine.java | 7 +- .../robin/RobinEngineIntegrationTest.java | 132 ++++++++++-------- .../indices/stats/SimpleIndexStatsTests.java | 2 + 3 files changed, 78 insertions(+), 63 deletions(-) diff --git a/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java b/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java index 5735370f018..46a925a38f0 100644 --- a/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java +++ b/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java @@ -1118,14 +1118,11 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine { } } - static final boolean allowRamBytesUsed; - + /*this is only used by one test right now and shoudl go away entirely once we update lucene*/ + private static boolean allowRamBytesUsed = false; static { assert Version.CURRENT.luceneVersion == org.apache.lucene.util.Version.LUCENE_46 : "when upgrading to a new lucene version, check if ramBytes is fixed, see https://issues.apache.org/jira/browse/LUCENE-5373"; - boolean xAllowRamBytesUsed = false; - assert xAllowRamBytesUsed = true; - allowRamBytesUsed = xAllowRamBytesUsed; } private long getReaderRamBytesUsed(AtomicReaderContext reader) { diff --git a/src/test/java/org/elasticsearch/index/engine/robin/RobinEngineIntegrationTest.java b/src/test/java/org/elasticsearch/index/engine/robin/RobinEngineIntegrationTest.java index e13005d2395..eb3c2dad0a6 100644 --- a/src/test/java/org/elasticsearch/index/engine/robin/RobinEngineIntegrationTest.java +++ b/src/test/java/org/elasticsearch/index/engine/robin/RobinEngineIntegrationTest.java @@ -38,6 +38,7 @@ import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.hamcrest.Matchers; import org.junit.Test; +import java.lang.reflect.Field; import java.util.Collection; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -47,74 +48,89 @@ public class RobinEngineIntegrationTest extends ElasticsearchIntegrationTest { @Test public void testSettingLoadBloomFilterDefaultTrue() throws Exception { - client().admin().indices().prepareCreate("test").setSettings(ImmutableSettings.builder().put("number_of_replicas", 0).put("number_of_shards", 1)).get(); - client().prepareIndex("test", "foo").setSource("field", "foo").get(); - ensureGreen(); - refresh(); + Field allowRamBytesUsed = RobinEngine.class.getDeclaredField("allowRamBytesUsed"); + allowRamBytesUsed.setAccessible(true); + allowRamBytesUsed.set(RobinEngine.class, Boolean.TRUE); - IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get(); - final long segmentsMemoryWithBloom = stats.getTotal().getSegments().getMemoryInBytes(); - logger.info("segments with bloom: {}", segmentsMemoryWithBloom); + try { + client().admin().indices().prepareCreate("test").setSettings(ImmutableSettings.builder().put("number_of_replicas", 0).put("number_of_shards", 1)).get(); + client().prepareIndex("test", "foo").setSource("field", "foo").get(); + ensureGreen(); + refresh(); + IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get(); + final long segmentsMemoryWithBloom = stats.getTotal().getSegments().getMemoryInBytes(); + logger.info("segments with bloom: {}", segmentsMemoryWithBloom); - logger.info("updating the setting to unload bloom filters"); - client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(CodecService.INDEX_CODEC_BLOOM_LOAD, false)).get(); - logger.info("waiting for memory to match without blooms"); - awaitBusy(new Predicate() { - public boolean apply(Object o) { - IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get(); - long segmentsMemoryWithoutBloom = stats.getTotal().getSegments().getMemoryInBytes(); - logger.info("trying segments without bloom: {}", segmentsMemoryWithoutBloom); - return segmentsMemoryWithoutBloom == (segmentsMemoryWithBloom - BloomFilter.Factory.DEFAULT.createFilter(1).getSizeInBytes()); - } - }); + logger.info("updating the setting to unload bloom filters"); + client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(CodecService.INDEX_CODEC_BLOOM_LOAD, false)).get(); + logger.info("waiting for memory to match without blooms"); + awaitBusy(new Predicate() { + public boolean apply(Object o) { + IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get(); + long segmentsMemoryWithoutBloom = stats.getTotal().getSegments().getMemoryInBytes(); + logger.info("trying segments without bloom: {}", segmentsMemoryWithoutBloom); + return segmentsMemoryWithoutBloom == (segmentsMemoryWithBloom - BloomFilter.Factory.DEFAULT.createFilter(1).getSizeInBytes()); + } + }); - logger.info("updating the setting to load bloom filters"); - client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(CodecService.INDEX_CODEC_BLOOM_LOAD, true)).get(); - logger.info("waiting for memory to match with blooms"); - awaitBusy(new Predicate() { - public boolean apply(Object o) { - IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get(); - long newSegmentsMemoryWithBloom = stats.getTotal().getSegments().getMemoryInBytes(); - logger.info("trying segments with bloom: {}", newSegmentsMemoryWithBloom); - return newSegmentsMemoryWithBloom == segmentsMemoryWithBloom; - } - }); + logger.info("updating the setting to load bloom filters"); + client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(CodecService.INDEX_CODEC_BLOOM_LOAD, true)).get(); + logger.info("waiting for memory to match with blooms"); + awaitBusy(new Predicate() { + public boolean apply(Object o) { + IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get(); + long newSegmentsMemoryWithBloom = stats.getTotal().getSegments().getMemoryInBytes(); + logger.info("trying segments with bloom: {}", newSegmentsMemoryWithBloom); + return newSegmentsMemoryWithBloom == segmentsMemoryWithBloom; + } + }); + } finally { + allowRamBytesUsed.set(RobinEngine.class, Boolean.FALSE); + } } @Test public void testSettingLoadBloomFilterDefaultFalse() throws Exception { - client().admin().indices().prepareCreate("test").setSettings(ImmutableSettings.builder().put("number_of_replicas", 0).put("number_of_shards", 1).put(CodecService.INDEX_CODEC_BLOOM_LOAD, false)).get(); - client().prepareIndex("test", "foo").setSource("field", "foo").get(); - ensureGreen(); - refresh(); + Field allowRamBytesUsed = RobinEngine.class.getDeclaredField("allowRamBytesUsed"); + allowRamBytesUsed.setAccessible(true); + allowRamBytesUsed.set(RobinEngine.class, Boolean.TRUE); + try { + client().admin().indices().prepareCreate("test").setSettings(ImmutableSettings.builder().put("number_of_replicas", 0).put("number_of_shards", 1).put(CodecService.INDEX_CODEC_BLOOM_LOAD, false)).get(); + client().prepareIndex("test", "foo").setSource("field", "foo").get(); + ensureGreen(); + refresh(); - IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get(); - final long segmentsMemoryWithoutBloom = stats.getTotal().getSegments().getMemoryInBytes(); - logger.info("segments without bloom: {}", segmentsMemoryWithoutBloom); + IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get(); + final long segmentsMemoryWithoutBloom = stats.getTotal().getSegments().getMemoryInBytes(); + logger.info("segments without bloom: {}", segmentsMemoryWithoutBloom); - logger.info("updating the setting to load bloom filters"); - client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(CodecService.INDEX_CODEC_BLOOM_LOAD, true)).get(); - logger.info("waiting for memory to match with blooms"); - awaitBusy(new Predicate() { - public boolean apply(Object o) { - IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get(); - long segmentsMemoryWithBloom = stats.getTotal().getSegments().getMemoryInBytes(); - logger.info("trying segments with bloom: {}", segmentsMemoryWithoutBloom); - return segmentsMemoryWithoutBloom == (segmentsMemoryWithBloom - BloomFilter.Factory.DEFAULT.createFilter(1).getSizeInBytes()); - } - }); + logger.info("updating the setting to load bloom filters"); + client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(CodecService.INDEX_CODEC_BLOOM_LOAD, true)).get(); + logger.info("waiting for memory to match with blooms"); + awaitBusy(new Predicate() { + public boolean apply(Object o) { + IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get(); + long segmentsMemoryWithBloom = stats.getTotal().getSegments().getMemoryInBytes(); + logger.info("trying segments with bloom: {}", segmentsMemoryWithoutBloom); + return segmentsMemoryWithoutBloom == (segmentsMemoryWithBloom - BloomFilter.Factory.DEFAULT.createFilter(1).getSizeInBytes()); + } + }); + + logger.info("updating the setting to unload bloom filters"); + client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(CodecService.INDEX_CODEC_BLOOM_LOAD, false)).get(); + logger.info("waiting for memory to match without blooms"); + awaitBusy(new Predicate() { + public boolean apply(Object o) { + IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get(); + long newSegmentsMemoryWithoutBloom = stats.getTotal().getSegments().getMemoryInBytes(); + logger.info("trying segments without bloom: {}", newSegmentsMemoryWithoutBloom); + return newSegmentsMemoryWithoutBloom == segmentsMemoryWithoutBloom; + } + }); + } finally { + allowRamBytesUsed.set(RobinEngine.class, Boolean.FALSE); + } - logger.info("updating the setting to unload bloom filters"); - client().admin().indices().prepareUpdateSettings("test").setSettings(ImmutableSettings.builder().put(CodecService.INDEX_CODEC_BLOOM_LOAD, false)).get(); - logger.info("waiting for memory to match without blooms"); - awaitBusy(new Predicate() { - public boolean apply(Object o) { - IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get(); - long newSegmentsMemoryWithoutBloom = stats.getTotal().getSegments().getMemoryInBytes(); - logger.info("trying segments without bloom: {}", newSegmentsMemoryWithoutBloom); - return newSegmentsMemoryWithoutBloom == segmentsMemoryWithoutBloom; - } - }); } @Test diff --git a/src/test/java/org/elasticsearch/indices/stats/SimpleIndexStatsTests.java b/src/test/java/org/elasticsearch/indices/stats/SimpleIndexStatsTests.java index 09f26f6c79f..e19dd587ba9 100644 --- a/src/test/java/org/elasticsearch/indices/stats/SimpleIndexStatsTests.java +++ b/src/test/java/org/elasticsearch/indices/stats/SimpleIndexStatsTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.indices.stats; +import org.apache.lucene.util.Version; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; @@ -207,6 +208,7 @@ public class SimpleIndexStatsTests extends ElasticsearchIntegrationTest { assertThat(stats.getTotal().getSegments(), notNullValue()); assertThat(stats.getTotal().getSegments().getCount(), equalTo(10l)); + assumeTrue(org.elasticsearch.Version.CURRENT.luceneVersion != Version.LUCENE_46); assertThat(stats.getTotal().getSegments().getMemoryInBytes(), greaterThan(0l)); }