convert index.max_result_window
This commit is contained in:
Simon Willnauer
parent
3512bea105
commit
3902ea439d
|
|
@ -172,7 +172,6 @@ public class ClusterModule extends AbstractModule {
|
|||
registerIndexDynamicSetting(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, Validator.BYTES_SIZE);
|
||||
registerIndexDynamicSetting(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED, Validator.BOOLEAN);
|
||||
registerIndexDynamicSetting(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, Validator.TIME);
|
||||
registerIndexDynamicSetting(DefaultSearchContext.MAX_RESULT_WINDOW, Validator.POSITIVE_INTEGER);
|
||||
}
|
||||
|
||||
public void registerIndexDynamicSetting(String setting, Validator validator) {
|
||||
|
|
|
|||
|
|
@ -61,6 +61,15 @@ public final class IndexSettings {
|
|||
public static final String INDEX_TRANSLOG_SYNC_INTERVAL = "index.translog.sync_interval";
|
||||
public static final Setting<Translog.Durability> INDEX_TRANSLOG_DURABILITY_SETTING = new Setting<>("index.translog.durability", Translog.Durability.REQUEST.name(), (value) -> Translog.Durability.valueOf(value.toUpperCase(Locale.ROOT)), true, Setting.Scope.INDEX);
|
||||
public static final Setting<Boolean> INDEX_WARMER_ENABLED_SETTING = Setting.boolSetting("index.warmer.enabled", true, true, Setting.Scope.INDEX);
|
||||
/**
|
||||
* Index setting describing the maximum value of from + size on a query.
|
||||
* The Default maximum value of from + size on a query is 10,000. This was chosen as
|
||||
* a conservative default as it is sure to not cause trouble. Users can
|
||||
* certainly profile their cluster and decide to set it to 100,000
|
||||
* safely. 1,000,000 is probably way to high for any cluster to set
|
||||
* safely.
|
||||
*/
|
||||
public static final Setting<Integer> MAX_RESULT_WINDOW_SETTING = Setting.intSetting("index.max_result_window", 10000, 1, true, Setting.Scope.INDEX);
|
||||
public static final TimeValue DEFAULT_REFRESH_INTERVAL = new TimeValue(1, TimeUnit.SECONDS);
|
||||
public static final Setting<TimeValue> INDEX_REFRESH_INTERVAL_SETTING = Setting.timeSetting("index.refresh_interval", DEFAULT_REFRESH_INTERVAL, new TimeValue(-1, TimeUnit.SECONDS), true, Setting.Scope.INDEX);
|
||||
public static final String INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE = "index.translog.flush_threshold_size";
|
||||
|
|
@ -99,6 +108,7 @@ public final class IndexSettings {
|
|||
private final ScopedSettings scopedSettings;
|
||||
private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis();
|
||||
private volatile boolean warmerEnabled;
|
||||
private volatile int maxResultWindow;
|
||||
|
||||
public static Set<Setting<?>> BUILT_IN_CLUSTER_SETTINGS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
|
||||
IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING,
|
||||
|
|
@ -108,10 +118,12 @@ public final class IndexSettings {
|
|||
MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING,
|
||||
IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING,
|
||||
IndexSettings.INDEX_WARMER_ENABLED_SETTING,
|
||||
IndexSettings.INDEX_REFRESH_INTERVAL_SETTING
|
||||
IndexSettings.INDEX_REFRESH_INTERVAL_SETTING,
|
||||
IndexSettings.MAX_RESULT_WINDOW_SETTING
|
||||
)));
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Returns the default search field for this index.
|
||||
*/
|
||||
|
|
@ -196,6 +208,8 @@ public final class IndexSettings {
|
|||
gcDeletesInMillis = settings.getAsTime(IndexSettings.INDEX_GC_DELETES_SETTING, DEFAULT_GC_DELETES).getMillis();
|
||||
warmerEnabled = scopedSettings.get(INDEX_WARMER_ENABLED_SETTING);
|
||||
scopedSettings.addSettingsUpdateConsumer(INDEX_WARMER_ENABLED_SETTING, this::setEnableWarmer);
|
||||
maxResultWindow = scopedSettings.get(MAX_RESULT_WINDOW_SETTING);
|
||||
scopedSettings.addSettingsUpdateConsumer(MAX_RESULT_WINDOW_SETTING, this::setMaxResultWindow);
|
||||
this.mergePolicyConfig = new MergePolicyConfig(logger, settings);
|
||||
assert indexNameMatcher.test(indexMetaData.getIndex());
|
||||
|
||||
|
|
@ -410,6 +424,18 @@ public final class IndexSettings {
|
|||
*/
|
||||
public MergeSchedulerConfig getMergeSchedulerConfig() { return mergeSchedulerConfig; }
|
||||
|
||||
/**
|
||||
* Returns the max result window for search requests, describing the maximum value of from + size on a query.
|
||||
*/
|
||||
public int getMaxResultWindow() {
|
||||
return this.maxResultWindow;
|
||||
}
|
||||
|
||||
private void setMaxResultWindow(int maxResultWindow) {
|
||||
this.maxResultWindow = maxResultWindow;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the GC deletes cycle in milliseconds.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ import org.elasticsearch.common.lucene.search.function.WeightFactorFunction;
|
|||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.common.util.BigArrays;
|
||||
import org.elasticsearch.index.IndexService;
|
||||
import org.elasticsearch.index.IndexSettings;
|
||||
import org.elasticsearch.index.analysis.AnalysisService;
|
||||
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
|
||||
import org.elasticsearch.index.engine.Engine;
|
||||
|
|
@ -78,20 +79,6 @@ import java.util.Map;
|
|||
*
|
||||
*/
|
||||
public class DefaultSearchContext extends SearchContext {
|
||||
/**
|
||||
* Index setting describing the maximum value of from + size on a query.
|
||||
*/
|
||||
public static final String MAX_RESULT_WINDOW = "index.max_result_window";
|
||||
public static class Defaults {
|
||||
/**
|
||||
* Default maximum value of from + size on a query. 10,000 was chosen as
|
||||
* a conservative default as it is sure to not cause trouble. Users can
|
||||
* certainly profile their cluster and decide to set it to 100,000
|
||||
* safely. 1,000,000 is probably way to high for any cluster to set
|
||||
* safely.
|
||||
*/
|
||||
public static final int MAX_RESULT_WINDOW = 10000;
|
||||
}
|
||||
|
||||
private final long id;
|
||||
private final ShardSearchRequest request;
|
||||
|
|
@ -202,13 +189,13 @@ public class DefaultSearchContext extends SearchContext {
|
|||
long resultWindow = from + size;
|
||||
// We need settingsService's view of the settings because its dynamic.
|
||||
// indexService's isn't.
|
||||
int maxResultWindow = indexService.getIndexSettings().getSettings().getAsInt(MAX_RESULT_WINDOW, Defaults.MAX_RESULT_WINDOW);
|
||||
int maxResultWindow = indexService.getIndexSettings().getMaxResultWindow();
|
||||
|
||||
if (resultWindow > maxResultWindow) {
|
||||
throw new QueryPhaseExecutionException(this,
|
||||
"Result window is too large, from + size must be less than or equal to: [" + maxResultWindow + "] but was ["
|
||||
+ resultWindow + "]. See the scroll api for a more efficient way to request large data sets. "
|
||||
+ "This limit can be set by changing the [" + DefaultSearchContext.MAX_RESULT_WINDOW
|
||||
+ "This limit can be set by changing the [" + IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey()
|
||||
+ "] index level parameter.");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -178,9 +178,6 @@ public class IndexSettingsTests extends ESTestCase {
|
|||
assertFalse(settings.isWarmerEnabled());
|
||||
settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.INDEX_WARMER_ENABLED_SETTING.getKey(), "true").build()));
|
||||
assertTrue(settings.isWarmerEnabled());
|
||||
|
||||
assertEquals(Translog.Durability.REQUEST, settings.getTranslogDurability());
|
||||
|
||||
metaData = newIndexMeta("index", Settings.settingsBuilder()
|
||||
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
|
||||
.build());
|
||||
|
|
@ -210,5 +207,23 @@ public class IndexSettingsTests extends ESTestCase {
|
|||
return refreshInterval;
|
||||
}
|
||||
|
||||
public void testMaxResultWindow() {
|
||||
IndexMetaData metaData = newIndexMeta("index", Settings.settingsBuilder()
|
||||
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
|
||||
.put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), 15)
|
||||
.build());
|
||||
IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
|
||||
assertEquals(15, settings.getMaxResultWindow());
|
||||
settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), 42).build()));
|
||||
assertEquals(42, settings.getMaxResultWindow());
|
||||
settings.updateIndexMetaData(newIndexMeta("index", Settings.EMPTY));
|
||||
assertEquals(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxResultWindow());
|
||||
|
||||
metaData = newIndexMeta("index", Settings.settingsBuilder()
|
||||
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
|
||||
.build());
|
||||
settings = new IndexSettings(metaData, Settings.EMPTY);
|
||||
assertEquals(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxResultWindow());
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
|
|||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.index.IndexSettings;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.rest.RestStatus;
|
||||
import org.elasticsearch.search.internal.DefaultSearchContext;
|
||||
|
|
@ -282,54 +283,54 @@ public class SimpleSearchIT extends ESIntegTestCase {
|
|||
createIndex("idx");
|
||||
indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
|
||||
|
||||
assertWindowFails(client().prepareSearch("idx").setFrom(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW));
|
||||
assertWindowFails(client().prepareSearch("idx").setSize(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW + 1));
|
||||
assertWindowFails(client().prepareSearch("idx").setSize(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW)
|
||||
.setFrom(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW));
|
||||
assertWindowFails(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)));
|
||||
assertWindowFails(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1));
|
||||
assertWindowFails(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY))
|
||||
.setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)));
|
||||
}
|
||||
|
||||
public void testLargeFromAndSizeSucceeds() throws Exception {
|
||||
createIndex("idx");
|
||||
indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
|
||||
|
||||
assertHitCount(client().prepareSearch("idx").setFrom(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW - 10).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setSize(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setSize(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW / 2)
|
||||
.setFrom(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW / 2 - 1).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) - 10).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) / 2)
|
||||
.setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) / 2 - 1).get(), 1);
|
||||
}
|
||||
|
||||
public void testTooLargeFromAndSizeOkBySetting() throws Exception {
|
||||
prepareCreate("idx").setSettings(DefaultSearchContext.MAX_RESULT_WINDOW, DefaultSearchContext.Defaults.MAX_RESULT_WINDOW * 2).get();
|
||||
prepareCreate("idx").setSettings(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2).get();
|
||||
indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
|
||||
|
||||
assertHitCount(client().prepareSearch("idx").setFrom(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setSize(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW + 1).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setSize(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW)
|
||||
.setFrom(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY))
|
||||
.setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1);
|
||||
}
|
||||
|
||||
public void testTooLargeFromAndSizeOkByDynamicSetting() throws Exception {
|
||||
createIndex("idx");
|
||||
assertAcked(client().admin().indices().prepareUpdateSettings("idx")
|
||||
.setSettings(
|
||||
Settings.builder().put(DefaultSearchContext.MAX_RESULT_WINDOW, DefaultSearchContext.Defaults.MAX_RESULT_WINDOW * 2))
|
||||
Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 2))
|
||||
.get());
|
||||
indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
|
||||
|
||||
assertHitCount(client().prepareSearch("idx").setFrom(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setSize(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW + 1).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setSize(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW)
|
||||
.setFrom(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) + 1).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY))
|
||||
.setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)).get(), 1);
|
||||
}
|
||||
|
||||
public void testTooLargeFromAndSizeBackwardsCompatibilityRecommendation() throws Exception {
|
||||
prepareCreate("idx").setSettings(DefaultSearchContext.MAX_RESULT_WINDOW, Integer.MAX_VALUE).get();
|
||||
prepareCreate("idx").setSettings(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), Integer.MAX_VALUE).get();
|
||||
indexRandom(true, client().prepareIndex("idx", "type").setSource("{}"));
|
||||
|
||||
assertHitCount(client().prepareSearch("idx").setFrom(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW * 10).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setSize(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW * 10).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setSize(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW * 10)
|
||||
.setFrom(DefaultSearchContext.Defaults.MAX_RESULT_WINDOW * 10).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1);
|
||||
assertHitCount(client().prepareSearch("idx").setSize(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10)
|
||||
.setFrom(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY) * 10).get(), 1);
|
||||
}
|
||||
|
||||
public void testQueryNumericFieldWithRegex() throws Exception {
|
||||
|
|
@ -350,7 +351,7 @@ public class SimpleSearchIT extends ESIntegTestCase {
|
|||
fail();
|
||||
} catch (SearchPhaseExecutionException e) {
|
||||
assertThat(e.toString(), containsString("Result window is too large, from + size must be less than or equal to: ["
|
||||
+ DefaultSearchContext.Defaults.MAX_RESULT_WINDOW));
|
||||
+ IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY)));
|
||||
assertThat(e.toString(), containsString("See the scroll api for a more efficient way to request large data sets"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue