Added a limit to from + size in top_hits and inner hits.

Relates to #11511
This commit is contained in:
Martijn van Groningen 2017-09-04 13:19:13 +02:00
parent 8f0369296f
commit 78e9c96d7f
No known key found for this signature in database
GPG Key ID: AB236F4FCF2AF12A
13 changed files with 264 additions and 11 deletions

View File

@ -18,7 +18,6 @@
*/ */
package org.elasticsearch.common.settings; package org.elasticsearch.common.settings;
import org.elasticsearch.index.IndexSortConfig;
import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.routing.UnassignedInfo;
import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
@ -27,6 +26,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocatio
import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexModule;
import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexSortConfig;
import org.elasticsearch.index.IndexingSlowLog; import org.elasticsearch.index.IndexingSlowLog;
import org.elasticsearch.index.MergePolicyConfig; import org.elasticsearch.index.MergePolicyConfig;
import org.elasticsearch.index.MergeSchedulerConfig; import org.elasticsearch.index.MergeSchedulerConfig;
@ -110,6 +110,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
IndexSettings.INDEX_WARMER_ENABLED_SETTING, IndexSettings.INDEX_WARMER_ENABLED_SETTING,
IndexSettings.INDEX_REFRESH_INTERVAL_SETTING, IndexSettings.INDEX_REFRESH_INTERVAL_SETTING,
IndexSettings.MAX_RESULT_WINDOW_SETTING, IndexSettings.MAX_RESULT_WINDOW_SETTING,
IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING,
IndexSettings.MAX_RESCORE_WINDOW_SETTING, IndexSettings.MAX_RESCORE_WINDOW_SETTING,
IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING, IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING,
IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING, IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING,

View File

@ -91,6 +91,13 @@ public final class IndexSettings {
*/ */
public static final Setting<Integer> MAX_RESULT_WINDOW_SETTING = public static final Setting<Integer> MAX_RESULT_WINDOW_SETTING =
Setting.intSetting("index.max_result_window", 10000, 1, Property.Dynamic, Property.IndexScope); Setting.intSetting("index.max_result_window", 10000, 1, Property.Dynamic, Property.IndexScope);
/**
* Index setting describing the maximum value of from + size on an individual inner hit definition or
* top hits aggregation. The default maximum of 100 is defensive for the reason that the number of inner hit responses
* and number of top hits buckets returned is unbounded. Profile your cluster when increasing this setting.
*/
public static final Setting<Integer> MAX_INNER_RESULT_WINDOW_SETTING =
Setting.intSetting("index.max_inner_result_window", 100, 1, Property.Dynamic, Property.IndexScope);
/** /**
* Index setting describing the maximum size of the rescore window. Defaults to {@link #MAX_RESULT_WINDOW_SETTING} * Index setting describing the maximum size of the rescore window. Defaults to {@link #MAX_RESULT_WINDOW_SETTING}
* because they both do the same thing: control the size of the heap of hits. * because they both do the same thing: control the size of the heap of hits.
@ -211,6 +218,7 @@ public final class IndexSettings {
private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis(); private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis();
private volatile boolean warmerEnabled; private volatile boolean warmerEnabled;
private volatile int maxResultWindow; private volatile int maxResultWindow;
private volatile int maxInnerResultWindow;
private volatile int maxAdjacencyMatrixFilters; private volatile int maxAdjacencyMatrixFilters;
private volatile int maxRescoreWindow; private volatile int maxRescoreWindow;
private volatile boolean TTLPurgeDisabled; private volatile boolean TTLPurgeDisabled;
@ -311,6 +319,7 @@ public final class IndexSettings {
gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis(); gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis();
warmerEnabled = scopedSettings.get(INDEX_WARMER_ENABLED_SETTING); warmerEnabled = scopedSettings.get(INDEX_WARMER_ENABLED_SETTING);
maxResultWindow = scopedSettings.get(MAX_RESULT_WINDOW_SETTING); maxResultWindow = scopedSettings.get(MAX_RESULT_WINDOW_SETTING);
maxInnerResultWindow = scopedSettings.get(MAX_INNER_RESULT_WINDOW_SETTING);
maxAdjacencyMatrixFilters = scopedSettings.get(MAX_ADJACENCY_MATRIX_FILTERS_SETTING); maxAdjacencyMatrixFilters = scopedSettings.get(MAX_ADJACENCY_MATRIX_FILTERS_SETTING);
maxRescoreWindow = scopedSettings.get(MAX_RESCORE_WINDOW_SETTING); maxRescoreWindow = scopedSettings.get(MAX_RESCORE_WINDOW_SETTING);
TTLPurgeDisabled = scopedSettings.get(INDEX_TTL_DISABLE_PURGE_SETTING); TTLPurgeDisabled = scopedSettings.get(INDEX_TTL_DISABLE_PURGE_SETTING);
@ -339,6 +348,7 @@ public final class IndexSettings {
scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_DURABILITY_SETTING, this::setTranslogDurability); scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_DURABILITY_SETTING, this::setTranslogDurability);
scopedSettings.addSettingsUpdateConsumer(INDEX_TTL_DISABLE_PURGE_SETTING, this::setTTLPurgeDisabled); scopedSettings.addSettingsUpdateConsumer(INDEX_TTL_DISABLE_PURGE_SETTING, this::setTTLPurgeDisabled);
scopedSettings.addSettingsUpdateConsumer(MAX_RESULT_WINDOW_SETTING, this::setMaxResultWindow); scopedSettings.addSettingsUpdateConsumer(MAX_RESULT_WINDOW_SETTING, this::setMaxResultWindow);
scopedSettings.addSettingsUpdateConsumer(MAX_INNER_RESULT_WINDOW_SETTING, this::setMaxInnerResultWindow);
scopedSettings.addSettingsUpdateConsumer(MAX_ADJACENCY_MATRIX_FILTERS_SETTING, this::setMaxAdjacencyMatrixFilters); scopedSettings.addSettingsUpdateConsumer(MAX_ADJACENCY_MATRIX_FILTERS_SETTING, this::setMaxAdjacencyMatrixFilters);
scopedSettings.addSettingsUpdateConsumer(MAX_RESCORE_WINDOW_SETTING, this::setMaxRescoreWindow); scopedSettings.addSettingsUpdateConsumer(MAX_RESCORE_WINDOW_SETTING, this::setMaxRescoreWindow);
scopedSettings.addSettingsUpdateConsumer(INDEX_WARMER_ENABLED_SETTING, this::setEnableWarmer); scopedSettings.addSettingsUpdateConsumer(INDEX_WARMER_ENABLED_SETTING, this::setEnableWarmer);
@ -564,6 +574,17 @@ public final class IndexSettings {
this.maxResultWindow = maxResultWindow; this.maxResultWindow = maxResultWindow;
} }
/**
* Returns the max result window for an individual inner hit definition or top hits aggregation.
*/
public int getMaxInnerResultWindow() {
return maxInnerResultWindow;
}
private void setMaxInnerResultWindow(int maxInnerResultWindow) {
this.maxInnerResultWindow = maxInnerResultWindow;
}
/** /**
* Returns the max number of filters in adjacency_matrix aggregation search requests * Returns the max number of filters in adjacency_matrix aggregation search requests
*/ */

View File

@ -19,7 +19,7 @@
package org.elasticsearch.index.query; package org.elasticsearch.index.query;
import org.elasticsearch.script.ScriptContext; import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.script.SearchScript; import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
@ -47,8 +47,21 @@ public abstract class InnerHitContextBuilder {
this.query = query; this.query = query;
} }
public abstract void build(SearchContext parentSearchContext, public final void build(SearchContext parentSearchContext, InnerHitsContext innerHitsContext) throws IOException {
InnerHitsContext innerHitsContext) throws IOException; long innerResultWindow = innerHitBuilder.getFrom() + innerHitBuilder.getSize();
int maxInnerResultWindow = parentSearchContext.mapperService().getIndexSettings().getMaxInnerResultWindow();
if (innerResultWindow > maxInnerResultWindow) {
throw new IllegalArgumentException(
"Inner result window is too large, the inner hit definition's [" + innerHitBuilder.getName() +
"]'s from + size must be less than or equal to: [" + maxInnerResultWindow + "] but was [" + innerResultWindow +
"]. This limit can be set by changing the [" + IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey() +
"] index level setting."
);
}
doBuild(parentSearchContext, innerHitsContext);
}
protected abstract void doBuild(SearchContext parentSearchContext, InnerHitsContext innerHitsContext) throws IOException;
public static void extractInnerHits(QueryBuilder query, Map<String, InnerHitContextBuilder> innerHitBuilders) { public static void extractInnerHits(QueryBuilder query, Map<String, InnerHitContextBuilder> innerHitBuilders) {
if (query instanceof AbstractQueryBuilder) { if (query instanceof AbstractQueryBuilder) {

View File

@ -336,7 +336,7 @@ public class NestedQueryBuilder extends AbstractQueryBuilder<NestedQueryBuilder>
} }
@Override @Override
public void build(SearchContext parentSearchContext, protected void doBuild(SearchContext parentSearchContext,
InnerHitsContext innerHitsContext) throws IOException { InnerHitsContext innerHitsContext) throws IOException {
QueryShardContext queryShardContext = parentSearchContext.getQueryShardContext(); QueryShardContext queryShardContext = parentSearchContext.getQueryShardContext();
ObjectMapper nestedObjectMapper = queryShardContext.getObjectMapper(path); ObjectMapper nestedObjectMapper = queryShardContext.getObjectMapper(path);

View File

@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.script.Script; import org.elasticsearch.script.Script;
import org.elasticsearch.script.SearchScript; import org.elasticsearch.script.SearchScript;
@ -529,6 +530,17 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
@Override @Override
protected TopHitsAggregatorFactory doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subfactoriesBuilder) protected TopHitsAggregatorFactory doBuild(SearchContext context, AggregatorFactory<?> parent, Builder subfactoriesBuilder)
throws IOException { throws IOException {
long innerResultWindow = from() + size();
int maxInnerResultWindow = context.mapperService().getIndexSettings().getMaxInnerResultWindow();
if (innerResultWindow > maxInnerResultWindow) {
throw new IllegalArgumentException(
"Top hits result window is too large, the top hits aggregator [" + name + "]'s from + size must be less " +
"than or equal to: [" + maxInnerResultWindow + "] but was [" + innerResultWindow +
"]. This limit can be set by changing the [" + IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey() +
"] index level setting."
);
}
List<ScriptFieldsContext.ScriptField> fields = new ArrayList<>(); List<ScriptFieldsContext.ScriptField> fields = new ArrayList<>();
if (scriptFields != null) { if (scriptFields != null) {
for (ScriptField field : scriptFields) { for (ScriptField field : scriptFields) {

View File

@ -290,6 +290,26 @@ public class IndexSettingsTests extends ESTestCase {
assertEquals(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxResultWindow()); assertEquals(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxResultWindow());
} }
public void testMaxInnerResultWindow() {
IndexMetaData metaData = newIndexMeta("index", Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), 200)
.build());
IndexSettings settings = new IndexSettings(metaData, Settings.EMPTY);
assertEquals(200, settings.getMaxInnerResultWindow());
settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(),
50).build()));
assertEquals(50, settings.getMaxInnerResultWindow());
settings.updateIndexMetaData(newIndexMeta("index", Settings.EMPTY));
assertEquals(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxInnerResultWindow());
metaData = newIndexMeta("index", Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.build());
settings = new IndexSettings(metaData, Settings.EMPTY);
assertEquals(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxInnerResultWindow());
}
public void testMaxAdjacencyMatrixFiltersSetting() { public void testMaxAdjacencyMatrixFiltersSetting() {
IndexMetaData metaData = newIndexMeta("index", Settings.builder() IndexMetaData metaData = newIndexMeta("index", Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)

View File

@ -139,8 +139,8 @@ public class InnerHitBuilderTests extends ESTestCase {
public static InnerHitBuilder randomInnerHits() { public static InnerHitBuilder randomInnerHits() {
InnerHitBuilder innerHits = new InnerHitBuilder(); InnerHitBuilder innerHits = new InnerHitBuilder();
innerHits.setName(randomAlphaOfLengthBetween(1, 16)); innerHits.setName(randomAlphaOfLengthBetween(1, 16));
innerHits.setFrom(randomIntBetween(0, 128)); innerHits.setFrom(randomIntBetween(0, 32));
innerHits.setSize(randomIntBetween(0, 128)); innerHits.setSize(randomIntBetween(0, 32));
innerHits.setExplain(randomBoolean()); innerHits.setExplain(randomBoolean());
innerHits.setVersion(randomBoolean()); innerHits.setVersion(randomBoolean());
innerHits.setTrackScores(randomBoolean()); innerHits.setTrackScores(randomBoolean());

View File

@ -26,6 +26,8 @@ import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.Version; import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.elasticsearch.index.search.ESToParentBlockJoinQuery; import org.elasticsearch.index.search.ESToParentBlockJoinQuery;
@ -41,6 +43,7 @@ import java.io.IOException;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta;
import static org.elasticsearch.index.query.InnerHitBuilderTests.randomInnerHits; import static org.elasticsearch.index.query.InnerHitBuilderTests.randomInnerHits;
import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.equalTo;
@ -325,6 +328,11 @@ public class NestedQueryBuilderTests extends AbstractQueryTestCase<NestedQueryBu
SearchContext searchContext = mock(SearchContext.class); SearchContext searchContext = mock(SearchContext.class);
when(searchContext.getQueryShardContext()).thenReturn(queryShardContext); when(searchContext.getQueryShardContext()).thenReturn(queryShardContext);
MapperService mapperService = mock(MapperService.class);
IndexSettings settings = new IndexSettings(newIndexMeta("index", Settings.EMPTY), Settings.EMPTY);
when(mapperService.getIndexSettings()).thenReturn(settings);
when(searchContext.mapperService()).thenReturn(mapperService);
InnerHitBuilder leafInnerHits = randomInnerHits(); InnerHitBuilder leafInnerHits = randomInnerHits();
NestedQueryBuilder query1 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None); NestedQueryBuilder query1 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None);
query1.innerHit(leafInnerHits); query1.innerHit(leafInnerHits);

View File

@ -28,6 +28,7 @@ import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.document.DocumentField;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.Plugin;
@ -942,7 +943,10 @@ public class TopHitsIT extends ESIntegTestCase {
} }
} }
public void testDontExplode() throws Exception { public void testUseMaxDocInsteadOfSize() throws Exception {
client().admin().indices().prepareUpdateSettings("idx")
.setSettings(Collections.singletonMap(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), ArrayUtil.MAX_ARRAY_LENGTH))
.get();
SearchResponse response = client() SearchResponse response = client()
.prepareSearch("idx") .prepareSearch("idx")
.addAggregation(terms("terms") .addAggregation(terms("terms")
@ -954,6 +958,67 @@ public class TopHitsIT extends ESIntegTestCase {
) )
.get(); .get();
assertNoFailures(response); assertNoFailures(response);
client().admin().indices().prepareUpdateSettings("idx")
.setSettings(Collections.singletonMap(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), null))
.get();
}
public void testTooHighResultWindow() throws Exception {
SearchResponse response = client()
.prepareSearch("idx")
.addAggregation(terms("terms")
.executionHint(randomExecutionHint())
.field(TERMS_AGGS_FIELD)
.subAggregation(
topHits("hits").from(50).size(10).sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))
)
)
.get();
assertNoFailures(response);
Exception e = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch("idx")
.addAggregation(terms("terms")
.executionHint(randomExecutionHint())
.field(TERMS_AGGS_FIELD)
.subAggregation(
topHits("hits").from(100).size(10).sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))
)
).get());
assertThat(e.getCause().getMessage(),
containsString("the top hits aggregator [hits]'s from + size must be less than or equal to: [100] but was [110]"));
e = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch("idx")
.addAggregation(terms("terms")
.executionHint(randomExecutionHint())
.field(TERMS_AGGS_FIELD)
.subAggregation(
topHits("hits").from(10).size(100).sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))
)
).get());
assertThat(e.getCause().getMessage(),
containsString("the top hits aggregator [hits]'s from + size must be less than or equal to: [100] but was [110]"));
client().admin().indices().prepareUpdateSettings("idx")
.setSettings(Collections.singletonMap(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), 110))
.get();
response = client().prepareSearch("idx")
.addAggregation(terms("terms")
.executionHint(randomExecutionHint())
.field(TERMS_AGGS_FIELD)
.subAggregation(
topHits("hits").from(100).size(10).sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))
)).get();
assertNoFailures(response);
response = client().prepareSearch("idx")
.addAggregation(terms("terms")
.executionHint(randomExecutionHint())
.field(TERMS_AGGS_FIELD)
.subAggregation(
topHits("hits").from(10).size(100).sort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))
)).get();
assertNoFailures(response);
client().admin().indices().prepareUpdateSettings("idx")
.setSettings(Collections.singletonMap(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), null))
.get();
} }
public void testNoStoredFields() throws Exception { public void testNoStoredFields() throws Exception {

View File

@ -23,10 +23,12 @@ import org.apache.lucene.search.join.ScoreMode;
import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.ArrayUtil;
import org.elasticsearch.Version; import org.elasticsearch.Version;
import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchPhaseExecutionException;
import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.InnerHitBuilder;
import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilder;
@ -629,8 +631,11 @@ public class InnerHitsIT extends ESIntegTestCase {
assertSearchHits(response, "1", "3"); assertSearchHits(response, "1", "3");
} }
public void testDontExplode() throws Exception { public void testUseMaxDocInsteadOfSize() throws Exception {
assertAcked(prepareCreate("index2").addMapping("type", "nested", "type=nested")); assertAcked(prepareCreate("index2").addMapping("type", "nested", "type=nested"));
client().admin().indices().prepareUpdateSettings("index2")
.setSettings(Collections.singletonMap(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), ArrayUtil.MAX_ARRAY_LENGTH))
.get();
client().prepareIndex("index2", "type", "1").setSource(jsonBuilder().startObject() client().prepareIndex("index2", "type", "1").setSource(jsonBuilder().startObject()
.startArray("nested") .startArray("nested")
.startObject() .startObject()
@ -650,4 +655,50 @@ public class InnerHitsIT extends ESIntegTestCase {
assertHitCount(response, 1); assertHitCount(response, 1);
} }
public void testTooHighResultWindow() throws Exception {
assertAcked(prepareCreate("index2").addMapping("type", "nested", "type=nested"));
client().prepareIndex("index2", "type", "1").setSource(jsonBuilder().startObject()
.startArray("nested")
.startObject()
.field("field", "value1")
.endObject()
.endArray()
.endObject())
.setRefreshPolicy(IMMEDIATE)
.get();
SearchResponse response = client().prepareSearch("index2")
.setQuery(nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg)
.innerHit(new InnerHitBuilder().setFrom(50).setSize(10).setName("_name")))
.get();
assertNoFailures(response);
assertHitCount(response, 1);
Exception e = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch("index2")
.setQuery(nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg)
.innerHit(new InnerHitBuilder().setFrom(100).setSize(10).setName("_name")))
.get());
assertThat(e.getCause().getMessage(),
containsString("the inner hit definition's [_name]'s from + size must be less than or equal to: [100] but was [110]"));
e = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch("index2")
.setQuery(nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg)
.innerHit(new InnerHitBuilder().setFrom(10).setSize(100).setName("_name")))
.get());
assertThat(e.getCause().getMessage(),
containsString("the inner hit definition's [_name]'s from + size must be less than or equal to: [100] but was [110]"));
client().admin().indices().prepareUpdateSettings("index2")
.setSettings(Collections.singletonMap(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), 110))
.get();
response = client().prepareSearch("index2")
.setQuery(nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg)
.innerHit(new InnerHitBuilder().setFrom(100).setSize(10).setName("_name")))
.get();
assertNoFailures(response);
response = client().prepareSearch("index2")
.setQuery(nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg)
.innerHit(new InnerHitBuilder().setFrom(10).setSize(100).setName("_name")))
.get();
assertNoFailures(response);
}
} }

View File

@ -121,6 +121,11 @@ specific index module:
<<search-request-scroll,Scroll>> or <<search-request-search-after,Search After>> for a more efficient alternative <<search-request-scroll,Scroll>> or <<search-request-search-after,Search After>> for a more efficient alternative
to raising this. to raising this.
`index.max_inner_result_window`::
The maximum value of `from + size` for inner hits definition and top hits aggregations to this index. Defaults to
`100`. Inner hits and top hits aggregation take heap memory and time proportional to `from + size` and this limits that memory.
`index.max_rescore_window`:: `index.max_rescore_window`::
The maximum value of `window_size` for `rescore` requests in searches of this index. The maximum value of `window_size` for `rescore` requests in searches of this index.

View File

@ -70,7 +70,7 @@ class ParentChildInnerHitContextBuilder extends InnerHitContextBuilder {
} }
@Override @Override
public void build(SearchContext parentSearchContext, InnerHitsContext innerHitsContext) throws IOException { protected void doBuild(SearchContext parentSearchContext, InnerHitsContext innerHitsContext) throws IOException {
if (parentSearchContext.mapperService().getIndexSettings().isSingleType()) { if (parentSearchContext.mapperService().getIndexSettings().isSingleType()) {
handleJoinFieldInnerHits(parentSearchContext, innerHitsContext); handleJoinFieldInnerHits(parentSearchContext, innerHitsContext);
} else { } else {

View File

@ -22,7 +22,9 @@ package org.elasticsearch.join.query;
import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.search.join.ScoreMode;
import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.ArrayUtil;
import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchPhaseExecutionException;
import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.InnerHitBuilder;
import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilder;
@ -506,13 +508,16 @@ public class InnerHitsIT extends ParentChildTestCase {
assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name2")); assertThat(response.getHits().getAt(0).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name2"));
} }
public void testDontExplode() throws Exception { public void testUseMaxDocInsteadOfSize() throws Exception {
if (legacy()) { if (legacy()) {
assertAcked(prepareCreate("index1").addMapping("child", "_parent", "type=parent")); assertAcked(prepareCreate("index1").addMapping("child", "_parent", "type=parent"));
} else { } else {
assertAcked(prepareCreate("index1") assertAcked(prepareCreate("index1")
.addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child"))); .addMapping("doc", buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent", "child")));
} }
client().admin().indices().prepareUpdateSettings("index1")
.setSettings(Collections.singletonMap(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), ArrayUtil.MAX_ARRAY_LENGTH))
.get();
List<IndexRequestBuilder> requests = new ArrayList<>(); List<IndexRequestBuilder> requests = new ArrayList<>();
requests.add(createIndexRequest("index1", "parent", "1", null)); requests.add(createIndexRequest("index1", "parent", "1", null));
requests.add(createIndexRequest("index1", "child", "2", "1", "field", "value1")); requests.add(createIndexRequest("index1", "child", "2", "1", "field", "value1"));
@ -585,4 +590,56 @@ public class InnerHitsIT extends ParentChildTestCase {
assertHitCount(response, 2); assertHitCount(response, 2);
assertSearchHits(response, "1", "3"); assertSearchHits(response, "1", "3");
} }
public void testTooHighResultWindow() throws Exception {
if (legacy()) {
assertAcked(prepareCreate("index1")
.addMapping("parent_type", "nested_type", "type=nested")
.addMapping("child_type", "_parent", "type=parent_type")
);
} else {
assertAcked(prepareCreate("index1")
.addMapping("doc", addFieldMappings(
buildParentJoinFieldMappingFromSimplifiedDef("join_field", true, "parent_type", "child_type"),
"nested_type", "nested"))
);
}
createIndexRequest("index1", "parent_type", "1", null, "nested_type", Collections.singletonMap("key", "value")).get();
createIndexRequest("index1", "child_type", "2", "1").get();
refresh();
SearchResponse response = client().prepareSearch("index1")
.setQuery(hasChildQuery("child_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true)
.innerHit(new InnerHitBuilder().setFrom(50).setSize(10).setName("_name")))
.get();
assertNoFailures(response);
assertHitCount(response, 1);
Exception e = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch("index1")
.setQuery(hasChildQuery("child_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true)
.innerHit(new InnerHitBuilder().setFrom(100).setSize(10).setName("_name")))
.get());
assertThat(e.getCause().getMessage(),
containsString("the inner hit definition's [_name]'s from + size must be less than or equal to: [100] but was [110]"));
e = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch("index1")
.setQuery(hasChildQuery("child_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true)
.innerHit(new InnerHitBuilder().setFrom(10).setSize(100).setName("_name")))
.get());
assertThat(e.getCause().getMessage(),
containsString("the inner hit definition's [_name]'s from + size must be less than or equal to: [100] but was [110]"));
client().admin().indices().prepareUpdateSettings("index1")
.setSettings(Collections.singletonMap(IndexSettings.MAX_INNER_RESULT_WINDOW_SETTING.getKey(), 110))
.get();
response = client().prepareSearch("index1")
.setQuery(hasChildQuery("child_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true)
.innerHit(new InnerHitBuilder().setFrom(100).setSize(10).setName("_name")))
.get();
assertNoFailures(response);
response = client().prepareSearch("index1")
.setQuery(hasChildQuery("child_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true)
.innerHit(new InnerHitBuilder().setFrom(10).setSize(100).setName("_name")))
.get();
assertNoFailures(response);
}
} }