Added a limit to from + size in top_hits and inner hits.
Relates to #11511
This commit is contained in:
parent
8f0369296f
commit
78e9c96d7f
|
@ -18,7 +18,6 @@
|
||||||
*/
|
*/
|
||||||
package org.elasticsearch.common.settings;
|
package org.elasticsearch.common.settings;
|
||||||
|
|
||||||
import org.elasticsearch.index.IndexSortConfig;
|
|
||||||
import org.elasticsearch.cluster.metadata.IndexMetaData;
|
import org.elasticsearch.cluster.metadata.IndexMetaData;
|
||||||
import org.elasticsearch.cluster.routing.UnassignedInfo;
|
import org.elasticsearch.cluster.routing.UnassignedInfo;
|
||||||
import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
|
import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
|
||||||
|
@ -27,6 +26,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocatio
|
||||||
import org.elasticsearch.common.settings.Setting.Property;
|
import org.elasticsearch.common.settings.Setting.Property;
|
||||||
import org.elasticsearch.index.IndexModule;
|
import org.elasticsearch.index.IndexModule;
|
||||||
import org.elasticsearch.index.IndexSettings;
|
import org.elasticsearch.index.IndexSettings;
|
||||||
|
import org.elasticsearch.index.IndexSortConfig;
|
||||||
import org.elasticsearch.index.IndexingSlowLog;
|
import org.elasticsearch.index.IndexingSlowLog;
|
||||||
import org.elasticsearch.index.MergePolicyConfig;
|
import org.elasticsearch.index.MergePolicyConfig;
|
||||||
import org.elasticsearch.index.MergeSchedulerConfig;
|
import org.elasticsearch.index.MergeSchedulerConfig;
|
||||||
|
@ -110,6 +110,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
|
||||||
IndexSettings.INDEX_WARMER_ENABLED_SETTING,
|
IndexSettings.INDEX_WARMER_ENABLED_SETTING,
|
||||||
IndexSettings.INDEX_REFRESH_INTERVAL_SETTING,
|
IndexSettings.INDEX_REFRESH_INTERVAL_SETTING,
|
||||||
IndexSettings.MAX_RESULT_WINDOW_SETTING,
|
IndexSettings.MAX_RESULT_WINDOW_SETTING,
|
||||||
|
IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING,
|
||||||
IndexSettings.MAX_RESCORE_WINDOW_SETTING,
|
IndexSettings.MAX_RESCORE_WINDOW_SETTING,
|
||||||
IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING,
|
IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING,
|
||||||
IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING,
|
IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING,
|
||||||
|
|
|
@ -91,6 +91,13 @@ public final class IndexSettings {
|
||||||
*/
|
*/
|
||||||
public static final Setting<Integer> MAX_RESULT_WINDOW_SETTING =
|
public static final Setting<Integer> MAX_RESULT_WINDOW_SETTING =
|
||||||
Setting.intSetting("index.max_result_window", 10000, 1, Property.Dynamic, Property.IndexScope);
|
Setting.intSetting("index.max_result_window", 10000, 1, Property.Dynamic, Property.IndexScope);
|
||||||
|
/**
|
||||||
|
* Index setting describing the maximum value of from + size on an individual inner hit definition or
|
||||||
|
* top hits aggregation. The default maximum of 100 is defensive for the reason that the number of inner hit responses
|
||||||
|
* and number of top hits buckets returned is unbounded. Profile your cluster when increasing this setting.
|
||||||
|
*/
|
||||||
|
public static final Setting<Integer> MAX_INNER_RESULT_WINDOW_SETTING =
|
||||||
|
Setting.intSetting("index.max_inner_result_window", 100, 1, Property.Dynamic, Property.IndexScope);
|
||||||
/**
|
/**
|
||||||
* Index setting describing the maximum size of the rescore window. Defaults to {@link #MAX_RESULT_WINDOW_SETTING}
|
* Index setting describing the maximum size of the rescore window. Defaults to {@link #MAX_RESULT_WINDOW_SETTING}
|
||||||
* because they both do the same thing: control the size of the heap of hits.
|
* because they both do the same thing: control the size of the heap of hits.
|
||||||
|
@ -211,6 +218,7 @@ public final class IndexSettings {
|
||||||
private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis();
|
private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis();
|
||||||
private volatile boolean warmerEnabled;
|
private volatile boolean warmerEnabled;
|
||||||
private volatile int maxResultWindow;
|
private volatile int maxResultWindow;
|
||||||
|
private volatile int maxInnerResultWindow;
|
||||||
private volatile int maxAdjacencyMatrixFilters;
|
private volatile int maxAdjacencyMatrixFilters;
|
||||||
private volatile int maxRescoreWindow;
|
private volatile int maxRescoreWindow;
|
||||||
private volatile boolean TTLPurgeDisabled;
|
private volatile boolean TTLPurgeDisabled;
|
||||||
|
@ -311,6 +319,7 @@ public final class IndexSettings {
|
||||||
gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis();
|
gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis();
|
||||||
warmerEnabled = scopedSettings.get(INDEX_WARMER_ENABLED_SETTING);
|
warmerEnabled = scopedSettings.get(INDEX_WARMER_ENABLED_SETTING);
|
||||||
maxResultWindow = scopedSettings.get(MAX_RESULT_WINDOW_SETTING);
|
maxResultWindow = scopedSettings.get(MAX_RESULT_WINDOW_SETTING);
|
||||||
|
maxInnerResultWindow = scopedSettings.get(MAX_INNER_RESULT_WINDOW_SETTING);
|
||||||
maxAdjacencyMatrixFilters = scopedSettings.get(MAX_ADJACENCY_MATRIX_FILTERS_SETTING);
|
maxAdjacencyMatrixFilters = scopedSettings.get(MAX_ADJACENCY_MATRIX_FILTERS_SETTING);
|
||||||
maxRescoreWindow = scopedSettings.get(MAX_RESCORE_WINDOW_SETTING);
|
maxRescoreWindow = scopedSettings.get(MAX_RESCORE_WINDOW_SETTING);
|
||||||
TTLPurgeDisabled = scopedSettings.get(INDEX_TTL_DISABLE_PURGE_SETTING);
|
TTLPurgeDisabled = scopedSettings.get(INDEX_TTL_DISABLE_PURGE_SETTING);
|
||||||
|
@ -339,6 +348,7 @@ public final class IndexSettings {
|
||||||
scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_DURABILITY_SETTING, this::setTranslogDurability);
|
scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_DURABILITY_SETTING, this::setTranslogDurability);
|
||||||
scopedSettings.addSettingsUpdateConsumer(INDEX_TTL_DISABLE_PURGE_SETTING, this::setTTLPurgeDisabled);
|
scopedSettings.addSettingsUpdateConsumer(INDEX_TTL_DISABLE_PURGE_SETTING, this::setTTLPurgeDisabled);
|
||||||
scopedSettings.addSettingsUpdateConsumer(MAX_RESULT_WINDOW_SETTING, this::setMaxResultWindow);
|
scopedSettings.addSettingsUpdateConsumer(MAX_RESULT_WINDOW_SETTING, this::setMaxResultWindow);
|
||||||
|
scopedSettings.addSettingsUpdateConsumer(MAX_INNER_RESULT_WINDOW_SETTING, this::setMaxInnerResultWindow);
|
||||||
scopedSettings.addSettingsUpdateConsumer(MAX_ADJACENCY_MATRIX_FILTERS_SETTING, this::setMaxAdjacencyMatrixFilters);
|
scopedSettings.addSettingsUpdateConsumer(MAX_ADJACENCY_MATRIX_FILTERS_SETTING, this::setMaxAdjacencyMatrixFilters);
|
||||||
scopedSettings.addSettingsUpdateConsumer(MAX_RESCORE_WINDOW_SETTING, this::setMaxRescoreWindow);
|
scopedSettings.addSettingsUpdateConsumer(MAX_RESCORE_WINDOW_SETTING, this::setMaxRescoreWindow);
|
||||||
scopedSettings.addSettingsUpdateConsumer(INDEX_WARMER_ENABLED_SETTING, this::setEnableWarmer);
|
scopedSettings.addSettingsUpdateConsumer(INDEX_WARMER_ENABLED_SETTING, this::setEnableWarmer);
|
||||||
|
@ -564,6 +574,17 @@ public final class IndexSettings {
|
||||||
this.maxResultWindow = maxResultWindow;
|
this.maxResultWindow = maxResultWindow;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the max result window for an individual inner hit definition or top hits aggregation.
|
||||||
|
*/
|
||||||
|
public int getMaxInnerResultWindow() {
|
||||||
|
return maxInnerResultWindow;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void setMaxInnerResultWindow(int maxInnerResultWindow) {
|
||||||
|
this.maxInnerResultWindow = maxInnerResultWindow;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the max number of filters in adjacency_matrix aggregation search requests
|
* Returns the max number of filters in adjacency_matrix aggregation search requests
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -19,7 +19,7 @@
|
||||||
|
|
||||||
package org.elasticsearch.index.query;
|
package org.elasticsearch.index.query;
|
||||||
|
|
||||||
import org.elasticsearch.script.ScriptContext;
|
import org.elasticsearch.index.IndexSettings;
|
||||||
import org.elasticsearch.script.SearchScript;
|
import org.elasticsearch.script.SearchScript;
|
||||||
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
import org.elasticsearch.search.builder.SearchSourceBuilder;
|
||||||
import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
|
import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
|
||||||
|
@ -47,8 +47,21 @@ public abstract class InnerHitContextBuilder {
|
||||||
this.query = query;
|
this.query = query;
|
||||||
}
|
}
|
||||||
|
|
||||||
public abstract void build(SearchContext parentSearchContext,
|
public final void build(SearchContext parentSearchContext, InnerHitsContext innerHitsContext) throws IOException {
|
||||||
InnerHitsContext innerHitsContext) throws IOException;
|
long innerResultWindow = innerHitBuilder.getFrom() + innerHitBuilder.getSize();
|
||||||
|
int maxInnerResultWindow = parentSearchContext.mapperService().getIndexSettings().getMaxInnerResultWindow();
|
||||||
|
if (innerResultWindow > maxInnerResultWindow) {
|
||||||
|
throw new IllegalArgumentException(
|
||||||
|
"Inner result window is too large, the inner hit definition's [" + innerHitBuilder.getName() +
|
||||||
|
"]'s from + size must be less than or equal to: [" + maxInnerResultWindow + "] but was [" + innerResultWindow +
|
||||||
|
"]. This limit can be set by changing the [" + IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey() +
|
||||||
|
"] index level setting."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
doBuild(parentSearchContext, innerHitsContext);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected abstract void doBuild(SearchContext parentSearchContext, InnerHitsContext innerHitsContext) throws IOException;
|
||||||
|
|
||||||
public static void extractInnerHits(QueryBuilder query, Map<String, InnerHitContextBuilder> innerHitBuilders) {
|
public static void extractInnerHits(QueryBuilder query, Map<String, InnerHitContextBuilder> innerHitBuilders) {
|
||||||
if (query instanceof AbstractQueryBuilder) {
|
if (query instanceof AbstractQueryBuilder) {
|
||||||
|
|
|
@ -336,7 +336,7 @@ public class NestedQueryBuilder extends AbstractQueryBuilder<NestedQueryBuilder>
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void build(SearchContext parentSearchContext,
|
protected void doBuild(SearchContext parentSearchContext,
|
||||||
InnerHitsContext innerHitsContext) throws IOException {
|
InnerHitsContext innerHitsContext) throws IOException {
|
||||||
QueryShardContext queryShardContext = parentSearchContext.getQueryShardContext();
|
QueryShardContext queryShardContext = parentSearchContext.getQueryShardContext();
|
||||||
ObjectMapper nestedObjectMapper = queryShardContext.getObjectMapper(path);
|
ObjectMapper nestedObjectMapper = queryShardContext.getObjectMapper(path);
|
||||||
|
|
|
@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
|
||||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
import org.elasticsearch.common.io.stream.StreamOutput;
|
||||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
import org.elasticsearch.common.xcontent.XContentParser;
|
import org.elasticsearch.common.xcontent.XContentParser;
|
||||||
|
import org.elasticsearch.index.IndexSettings;
|
||||||
import org.elasticsearch.index.query.QueryShardContext;
|
import org.elasticsearch.index.query.QueryShardContext;
|
||||||
import org.elasticsearch.script.Script;
|
import org.elasticsearch.script.Script;
|
||||||
import org.elasticsearch.script.SearchScript;
|
import org.elasticsearch.script.SearchScript;
|
||||||
|
@ -529,6 +530,17 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
|
||||||
@Override
|
@Override
|
||||||
protected TopHitsAggregatorFactory doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subfactoriesBuilder)
|
protected TopHitsAggregatorFactory doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subfactoriesBuilder)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
long innerResultWindow = from() + size();
|
||||||
|
int maxInnerResultWindow = context.mapperService().getIndexSettings().getMaxInnerResultWindow();
|
||||||
|
if (innerResultWindow > maxInnerResultWindow) {
|
||||||
|
throw new IllegalArgumentException(
|
||||||
|
"Top hits result window is too large, the top hits aggregator [" + name + "]'s from + size must be less " +
|
||||||
|
"than or equal to: [" + maxInnerResultWindow + "] but was [" + innerResultWindow +
|
||||||
|
"]. This limit can be set by changing the [" + IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey() +
|
||||||
|
"] index level setting."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
List<ScriptFieldsContext.ScriptField> fields = new ArrayList<>();
|
List<ScriptFieldsContext.ScriptField> fields = new ArrayList<>();
|
||||||
if (scriptFields != null) {
|
if (scriptFields != null) {
|
||||||
for (ScriptField field : scriptFields) {
|
for (ScriptField field : scriptFields) {
|
||||||
|
|
|
@ -290,6 +290,26 @@ public class IndexSettingsTests extends ESTestCase {
|
||||||
assertEquals(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxResultWindow());
|
assertEquals(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxResultWindow());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void testMaxInnerResultWindow() {
|
||||||
|
IndexMetaData metaData = newIndexMeta("index", Settings.builder()
|
||||||
|
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
|
||||||
|
.put(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), 200)
|
||||||
|
.build());
|
||||||
|
IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
|
||||||
|
assertEquals(200, settings.getMaxInnerResultWindow());
|
||||||
|
settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(),
|
||||||
|
50).build()));
|
||||||
|
assertEquals(50, settings.getMaxInnerResultWindow());
|
||||||
|
settings.updateIndexMetaData(newIndexMeta("index", Settings.EMPTY));
|
||||||
|
assertEquals(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxInnerResultWindow());
|
||||||
|
|
||||||
|
metaData = newIndexMeta("index", Settings.builder()
|
||||||
|
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
|
||||||
|
.build());
|
||||||
|
settings = new IndexSettings(metaData, Settings.EMPTY);
|
||||||
|
assertEquals(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxInnerResultWindow());
|
||||||
|
}
|
||||||
|
|
||||||
public void testMaxAdjacencyMatrixFiltersSetting() {
|
public void testMaxAdjacencyMatrixFiltersSetting() {
|
||||||
IndexMetaData metaData = newIndexMeta("index", Settings.builder()
|
IndexMetaData metaData = newIndexMeta("index", Settings.builder()
|
||||||
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
|
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
|
||||||
|
|
|
@ -139,8 +139,8 @@ public class InnerHitBuilderTests extends ESTestCase {
|
||||||
public static InnerHitBuilder randomInnerHits() {
|
public static InnerHitBuilder randomInnerHits() {
|
||||||
InnerHitBuilder innerHits = new InnerHitBuilder();
|
InnerHitBuilder innerHits = new InnerHitBuilder();
|
||||||
innerHits.setName(randomAlphaOfLengthBetween(1, 16));
|
innerHits.setName(randomAlphaOfLengthBetween(1, 16));
|
||||||
innerHits.setFrom(randomIntBetween(0, 128));
|
innerHits.setFrom(randomIntBetween(0, 32));
|
||||||
innerHits.setSize(randomIntBetween(0, 128));
|
innerHits.setSize(randomIntBetween(0, 32));
|
||||||
innerHits.setExplain(randomBoolean());
|
innerHits.setExplain(randomBoolean());
|
||||||
innerHits.setVersion(randomBoolean());
|
innerHits.setVersion(randomBoolean());
|
||||||
innerHits.setTrackScores(randomBoolean());
|
innerHits.setTrackScores(randomBoolean());
|
||||||
|
|
|
@ -26,6 +26,8 @@ import org.apache.lucene.search.join.ScoreMode;
|
||||||
import org.elasticsearch.Version;
|
import org.elasticsearch.Version;
|
||||||
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
|
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
|
||||||
import org.elasticsearch.common.compress.CompressedXContent;
|
import org.elasticsearch.common.compress.CompressedXContent;
|
||||||
|
import org.elasticsearch.common.settings.Settings;
|
||||||
|
import org.elasticsearch.index.IndexSettings;
|
||||||
import org.elasticsearch.index.mapper.MapperService;
|
import org.elasticsearch.index.mapper.MapperService;
|
||||||
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
|
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
|
||||||
import org.elasticsearch.index.search.ESToParentBlockJoinQuery;
|
import org.elasticsearch.index.search.ESToParentBlockJoinQuery;
|
||||||
|
@ -41,6 +43,7 @@ import java.io.IOException;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
|
import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta;
|
||||||
import static org.elasticsearch.index.query.InnerHitBuilderTests.randomInnerHits;
|
import static org.elasticsearch.index.query.InnerHitBuilderTests.randomInnerHits;
|
||||||
import static org.hamcrest.CoreMatchers.containsString;
|
import static org.hamcrest.CoreMatchers.containsString;
|
||||||
import static org.hamcrest.CoreMatchers.equalTo;
|
import static org.hamcrest.CoreMatchers.equalTo;
|
||||||
|
@ -325,6 +328,11 @@ public class NestedQueryBuilderTests extends AbstractQueryTestCase<NestedQueryBu
|
||||||
SearchContext searchContext = mock(SearchContext.class);
|
SearchContext searchContext = mock(SearchContext.class);
|
||||||
when(searchContext.getQueryShardContext()).thenReturn(queryShardContext);
|
when(searchContext.getQueryShardContext()).thenReturn(queryShardContext);
|
||||||
|
|
||||||
|
MapperService mapperService = mock(MapperService.class);
|
||||||
|
IndexSettings settings = new IndexSettings(newIndexMeta("index", Settings.EMPTY), Settings.EMPTY);
|
||||||
|
when(mapperService.getIndexSettings()).thenReturn(settings);
|
||||||
|
when(searchContext.mapperService()).thenReturn(mapperService);
|
||||||
|
|
||||||
InnerHitBuilder leafInnerHits = randomInnerHits();
|
InnerHitBuilder leafInnerHits = randomInnerHits();
|
||||||
NestedQueryBuilder query1 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None);
|
NestedQueryBuilder query1 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None);
|
||||||
query1.innerHit(leafInnerHits);
|
query1.innerHit(leafInnerHits);
|
||||||
|
|
|
@ -28,6 +28,7 @@ import org.elasticsearch.action.search.SearchType;
|
||||||
import org.elasticsearch.common.document.DocumentField;
|
import org.elasticsearch.common.document.DocumentField;
|
||||||
import org.elasticsearch.common.settings.Settings;
|
import org.elasticsearch.common.settings.Settings;
|
||||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
|
import org.elasticsearch.index.IndexSettings;
|
||||||
import org.elasticsearch.index.query.MatchAllQueryBuilder;
|
import org.elasticsearch.index.query.MatchAllQueryBuilder;
|
||||||
import org.elasticsearch.index.query.QueryBuilders;
|
import org.elasticsearch.index.query.QueryBuilders;
|
||||||
import org.elasticsearch.plugins.Plugin;
|
import org.elasticsearch.plugins.Plugin;
|
||||||
|
@ -942,7 +943,10 @@ public class TopHitsIT extends ESIntegTestCase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testDontExplode() throws Exception {
|
public void testUseMaxDocInsteadOfSize() throws Exception {
|
||||||
|
client().admin().indices().prepareUpdateSettings("idx")
|
||||||
|
.setSettings(Collections.singletonMap(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), ArrayUtil.MAX_ARRAY_LENGTH))
|
||||||
|
.get();
|
||||||
SearchResponse response = client()
|
SearchResponse response = client()
|
||||||
.prepareSearch("idx")
|
.prepareSearch("idx")
|
||||||
.addAggregation(terms("terms")
|
.addAggregation(terms("terms")
|
||||||
|
@ -954,6 +958,67 @@ public class TopHitsIT extends ESIntegTestCase {
|
||||||
)
|
)
|
||||||
.get();
|
.get();
|
||||||
assertNoFailures(response);
|
assertNoFailures(response);
|
||||||
|
client().admin().indices().prepareUpdateSettings("idx")
|
||||||
|
.setSettings(Collections.singletonMap(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), null))
|
||||||
|
.get();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testTooHighResultWindow() throws Exception {
|
||||||
|
SearchResponse response = client()
|
||||||
|
.prepareSearch("idx")
|
||||||
|
.addAggregation(terms("terms")
|
||||||
|
.executionHint(randomExecutionHint())
|
||||||
|
.field(TERMS_AGGS_FIELD)
|
||||||
|
.subAggregation(
|
||||||
|
topHits("hits").from(50).size(10).sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.get();
|
||||||
|
assertNoFailures(response);
|
||||||
|
|
||||||
|
Exception e = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch("idx")
|
||||||
|
.addAggregation(terms("terms")
|
||||||
|
.executionHint(randomExecutionHint())
|
||||||
|
.field(TERMS_AGGS_FIELD)
|
||||||
|
.subAggregation(
|
||||||
|
topHits("hits").from(100).size(10).sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))
|
||||||
|
)
|
||||||
|
).get());
|
||||||
|
assertThat(e.getCause().getMessage(),
|
||||||
|
containsString("the top hits aggregator [hits]'s from + size must be less than or equal to: [100] but was [110]"));
|
||||||
|
e = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch("idx")
|
||||||
|
.addAggregation(terms("terms")
|
||||||
|
.executionHint(randomExecutionHint())
|
||||||
|
.field(TERMS_AGGS_FIELD)
|
||||||
|
.subAggregation(
|
||||||
|
topHits("hits").from(10).size(100).sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))
|
||||||
|
)
|
||||||
|
).get());
|
||||||
|
assertThat(e.getCause().getMessage(),
|
||||||
|
containsString("the top hits aggregator [hits]'s from + size must be less than or equal to: [100] but was [110]"));
|
||||||
|
|
||||||
|
client().admin().indices().prepareUpdateSettings("idx")
|
||||||
|
.setSettings(Collections.singletonMap(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), 110))
|
||||||
|
.get();
|
||||||
|
response = client().prepareSearch("idx")
|
||||||
|
.addAggregation(terms("terms")
|
||||||
|
.executionHint(randomExecutionHint())
|
||||||
|
.field(TERMS_AGGS_FIELD)
|
||||||
|
.subAggregation(
|
||||||
|
topHits("hits").from(100).size(10).sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))
|
||||||
|
)).get();
|
||||||
|
assertNoFailures(response);
|
||||||
|
response = client().prepareSearch("idx")
|
||||||
|
.addAggregation(terms("terms")
|
||||||
|
.executionHint(randomExecutionHint())
|
||||||
|
.field(TERMS_AGGS_FIELD)
|
||||||
|
.subAggregation(
|
||||||
|
topHits("hits").from(10).size(100).sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))
|
||||||
|
)).get();
|
||||||
|
assertNoFailures(response);
|
||||||
|
client().admin().indices().prepareUpdateSettings("idx")
|
||||||
|
.setSettings(Collections.singletonMap(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), null))
|
||||||
|
.get();
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testNoStoredFields() throws Exception {
|
public void testNoStoredFields() throws Exception {
|
||||||
|
|
|
@ -23,10 +23,12 @@ import org.apache.lucene.search.join.ScoreMode;
|
||||||
import org.apache.lucene.util.ArrayUtil;
|
import org.apache.lucene.util.ArrayUtil;
|
||||||
import org.elasticsearch.Version;
|
import org.elasticsearch.Version;
|
||||||
import org.elasticsearch.action.index.IndexRequestBuilder;
|
import org.elasticsearch.action.index.IndexRequestBuilder;
|
||||||
|
import org.elasticsearch.action.search.SearchPhaseExecutionException;
|
||||||
import org.elasticsearch.action.search.SearchResponse;
|
import org.elasticsearch.action.search.SearchResponse;
|
||||||
import org.elasticsearch.cluster.health.ClusterHealthStatus;
|
import org.elasticsearch.cluster.health.ClusterHealthStatus;
|
||||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||||
import org.elasticsearch.common.xcontent.XContentType;
|
import org.elasticsearch.common.xcontent.XContentType;
|
||||||
|
import org.elasticsearch.index.IndexSettings;
|
||||||
import org.elasticsearch.index.query.BoolQueryBuilder;
|
import org.elasticsearch.index.query.BoolQueryBuilder;
|
||||||
import org.elasticsearch.index.query.InnerHitBuilder;
|
import org.elasticsearch.index.query.InnerHitBuilder;
|
||||||
import org.elasticsearch.index.query.QueryBuilder;
|
import org.elasticsearch.index.query.QueryBuilder;
|
||||||
|
@ -629,8 +631,11 @@ public class InnerHitsIT extends ESIntegTestCase {
|
||||||
assertSearchHits(response, "1", "3");
|
assertSearchHits(response, "1", "3");
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testDontExplode() throws Exception {
|
public void testUseMaxDocInsteadOfSize() throws Exception {
|
||||||
assertAcked(prepareCreate("index2").addMapping("type", "nested", "type=nested"));
|
assertAcked(prepareCreate("index2").addMapping("type", "nested", "type=nested"));
|
||||||
|
client().admin().indices().prepareUpdateSettings("index2")
|
||||||
|
.setSettings(Collections.singletonMap(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), ArrayUtil.MAX_ARRAY_LENGTH))
|
||||||
|
.get();
|
||||||
client().prepareIndex("index2", "type", "1").setSource(jsonBuilder().startObject()
|
client().prepareIndex("index2", "type", "1").setSource(jsonBuilder().startObject()
|
||||||
.startArray("nested")
|
.startArray("nested")
|
||||||
.startObject()
|
.startObject()
|
||||||
|
@ -650,4 +655,50 @@ public class InnerHitsIT extends ESIntegTestCase {
|
||||||
assertHitCount(response, 1);
|
assertHitCount(response, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void testTooHighResultWindow() throws Exception {
|
||||||
|
assertAcked(prepareCreate("index2").addMapping("type", "nested", "type=nested"));
|
||||||
|
client().prepareIndex("index2", "type", "1").setSource(jsonBuilder().startObject()
|
||||||
|
.startArray("nested")
|
||||||
|
.startObject()
|
||||||
|
.field("field", "value1")
|
||||||
|
.endObject()
|
||||||
|
.endArray()
|
||||||
|
.endObject())
|
||||||
|
.setRefreshPolicy(IMMEDIATE)
|
||||||
|
.get();
|
||||||
|
SearchResponse response = client().prepareSearch("index2")
|
||||||
|
.setQuery(nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg)
|
||||||
|
.innerHit(new InnerHitBuilder().setFrom(50).setSize(10).setName("_name")))
|
||||||
|
.get();
|
||||||
|
assertNoFailures(response);
|
||||||
|
assertHitCount(response, 1);
|
||||||
|
|
||||||
|
Exception e = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch("index2")
|
||||||
|
.setQuery(nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg)
|
||||||
|
.innerHit(new InnerHitBuilder().setFrom(100).setSize(10).setName("_name")))
|
||||||
|
.get());
|
||||||
|
assertThat(e.getCause().getMessage(),
|
||||||
|
containsString("the inner hit definition's [_name]'s from + size must be less than or equal to: [100] but was [110]"));
|
||||||
|
e = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch("index2")
|
||||||
|
.setQuery(nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg)
|
||||||
|
.innerHit(new InnerHitBuilder().setFrom(10).setSize(100).setName("_name")))
|
||||||
|
.get());
|
||||||
|
assertThat(e.getCause().getMessage(),
|
||||||
|
containsString("the inner hit definition's [_name]'s from + size must be less than or equal to: [100] but was [110]"));
|
||||||
|
|
||||||
|
client().admin().indices().prepareUpdateSettings("index2")
|
||||||
|
.setSettings(Collections.singletonMap(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), 110))
|
||||||
|
.get();
|
||||||
|
response = client().prepareSearch("index2")
|
||||||
|
.setQuery(nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg)
|
||||||
|
.innerHit(new InnerHitBuilder().setFrom(100).setSize(10).setName("_name")))
|
||||||
|
.get();
|
||||||
|
assertNoFailures(response);
|
||||||
|
response = client().prepareSearch("index2")
|
||||||
|
.setQuery(nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg)
|
||||||
|
.innerHit(new InnerHitBuilder().setFrom(10).setSize(100).setName("_name")))
|
||||||
|
.get();
|
||||||
|
assertNoFailures(response);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -121,6 +121,11 @@ specific index module:
|
||||||
<<search-request-scroll,Scroll>> or <<search-request-search-after,Search After>> for a more efficient alternative
|
<<search-request-scroll,Scroll>> or <<search-request-search-after,Search After>> for a more efficient alternative
|
||||||
to raising this.
|
to raising this.
|
||||||
|
|
||||||
|
`index.max_inner_result_window`::
|
||||||
|
|
||||||
|
The maximum value of `from + size` for inner hits definition and top hits aggregations to this index. Defaults to
|
||||||
|
`100`. Inner hits and top hits aggregation take heap memory and time proportional to `from + size` and this limits that memory.
|
||||||
|
|
||||||
`index.max_rescore_window`::
|
`index.max_rescore_window`::
|
||||||
|
|
||||||
The maximum value of `window_size` for `rescore` requests in searches of this index.
|
The maximum value of `window_size` for `rescore` requests in searches of this index.
|
||||||
|
|
|
@ -70,7 +70,7 @@ class ParentChildInnerHitContextBuilder extends InnerHitContextBuilder {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void build(SearchContext parentSearchContext, InnerHitsContext innerHitsContext) throws IOException {
|
protected void doBuild(SearchContext parentSearchContext, InnerHitsContext innerHitsContext) throws IOException {
|
||||||
if (parentSearchContext.mapperService().getIndexSettings().isSingleType()) {
|
if (parentSearchContext.mapperService().getIndexSettings().isSingleType()) {
|
||||||
handleJoinFieldInnerHits(parentSearchContext, innerHitsContext);
|
handleJoinFieldInnerHits(parentSearchContext, innerHitsContext);
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -22,7 +22,9 @@ package org.elasticsearch.join.query;
|
||||||
import org.apache.lucene.search.join.ScoreMode;
|
import org.apache.lucene.search.join.ScoreMode;
|
||||||
import org.apache.lucene.util.ArrayUtil;
|
import org.apache.lucene.util.ArrayUtil;
|
||||||
import org.elasticsearch.action.index.IndexRequestBuilder;
|
import org.elasticsearch.action.index.IndexRequestBuilder;
|
||||||
|
import org.elasticsearch.action.search.SearchPhaseExecutionException;
|
||||||
import org.elasticsearch.action.search.SearchResponse;
|
import org.elasticsearch.action.search.SearchResponse;
|
||||||
|
import org.elasticsearch.index.IndexSettings;
|
||||||
import org.elasticsearch.index.query.BoolQueryBuilder;
|
import org.elasticsearch.index.query.BoolQueryBuilder;
|
||||||
import org.elasticsearch.index.query.InnerHitBuilder;
|
import org.elasticsearch.index.query.InnerHitBuilder;
|
||||||
import org.elasticsearch.index.query.QueryBuilder;
|
import org.elasticsearch.index.query.QueryBuilder;
|
||||||
|
@ -506,13 +508,16 @@ public class InnerHitsIT extends ParentChildTestCase {
|
||||||
assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name2"));
|
assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name2"));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testDontExplode() throws Exception {
|
public void testUseMaxDocInsteadOfSize() throws Exception {
|
||||||
if (legacy()) {
|
if (legacy()) {
|
||||||
assertAcked(prepareCreate("index1").addMapping("child", "_parent", "type=parent"));
|
assertAcked(prepareCreate("index1").addMapping("child", "_parent", "type=parent"));
|
||||||
} else {
|
} else {
|
||||||
assertAcked(prepareCreate("index1")
|
assertAcked(prepareCreate("index1")
|
||||||
.addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
|
.addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
|
||||||
}
|
}
|
||||||
|
client().admin().indices().prepareUpdateSettings("index1")
|
||||||
|
.setSettings(Collections.singletonMap(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), ArrayUtil.MAX_ARRAY_LENGTH))
|
||||||
|
.get();
|
||||||
List<IndexRequestBuilder> requests = new ArrayList<>();
|
List<IndexRequestBuilder> requests = new ArrayList<>();
|
||||||
requests.add(createIndexRequest("index1", "parent", "1", null));
|
requests.add(createIndexRequest("index1", "parent", "1", null));
|
||||||
requests.add(createIndexRequest("index1", "child", "2", "1", "field", "value1"));
|
requests.add(createIndexRequest("index1", "child", "2", "1", "field", "value1"));
|
||||||
|
@ -585,4 +590,56 @@ public class InnerHitsIT extends ParentChildTestCase {
|
||||||
assertHitCount(response, 2);
|
assertHitCount(response, 2);
|
||||||
assertSearchHits(response, "1", "3");
|
assertSearchHits(response, "1", "3");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void testTooHighResultWindow() throws Exception {
|
||||||
|
if (legacy()) {
|
||||||
|
assertAcked(prepareCreate("index1")
|
||||||
|
.addMapping("parent_type", "nested_type", "type=nested")
|
||||||
|
.addMapping("child_type", "_parent", "type=parent_type")
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
assertAcked(prepareCreate("index1")
|
||||||
|
.addMapping("doc", addFieldMappings(
|
||||||
|
buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent_type", "child_type"),
|
||||||
|
"nested_type", "nested"))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
createIndexRequest("index1", "parent_type", "1", null, "nested_type", Collections.singletonMap("key", "value")).get();
|
||||||
|
createIndexRequest("index1", "child_type", "2", "1").get();
|
||||||
|
refresh();
|
||||||
|
|
||||||
|
SearchResponse response = client().prepareSearch("index1")
|
||||||
|
.setQuery(hasChildQuery("child_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true)
|
||||||
|
.innerHit(new InnerHitBuilder().setFrom(50).setSize(10).setName("_name")))
|
||||||
|
.get();
|
||||||
|
assertNoFailures(response);
|
||||||
|
assertHitCount(response, 1);
|
||||||
|
|
||||||
|
Exception e = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch("index1")
|
||||||
|
.setQuery(hasChildQuery("child_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true)
|
||||||
|
.innerHit(new InnerHitBuilder().setFrom(100).setSize(10).setName("_name")))
|
||||||
|
.get());
|
||||||
|
assertThat(e.getCause().getMessage(),
|
||||||
|
containsString("the inner hit definition's [_name]'s from + size must be less than or equal to: [100] but was [110]"));
|
||||||
|
e = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch("index1")
|
||||||
|
.setQuery(hasChildQuery("child_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true)
|
||||||
|
.innerHit(new InnerHitBuilder().setFrom(10).setSize(100).setName("_name")))
|
||||||
|
.get());
|
||||||
|
assertThat(e.getCause().getMessage(),
|
||||||
|
containsString("the inner hit definition's [_name]'s from + size must be less than or equal to: [100] but was [110]"));
|
||||||
|
|
||||||
|
client().admin().indices().prepareUpdateSettings("index1")
|
||||||
|
.setSettings(Collections.singletonMap(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), 110))
|
||||||
|
.get();
|
||||||
|
response = client().prepareSearch("index1")
|
||||||
|
.setQuery(hasChildQuery("child_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true)
|
||||||
|
.innerHit(new InnerHitBuilder().setFrom(100).setSize(10).setName("_name")))
|
||||||
|
.get();
|
||||||
|
assertNoFailures(response);
|
||||||
|
response = client().prepareSearch("index1")
|
||||||
|
.setQuery(hasChildQuery("child_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true)
|
||||||
|
.innerHit(new InnerHitBuilder().setFrom(10).setSize(100).setName("_name")))
|
||||||
|
.get();
|
||||||
|
assertNoFailures(response);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue