Cwikius-Spring/spring-data-elasticsearch
1
0
Fork 0
mirror of https://github.com/spring-projects/spring-data-elasticsearch.git synced 2025-06-21 11:32:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

DATAES-518 - Use scroll for unpaged find operations in ReactiveElasticsearchTemplate.

Browse Source 
We now use scroll instead of a max page size for unpaged queries.

Original pull request: #233.
This commit is contained in:
Christoph Strobl 2018-12-11 15:20:43 +01:00 committed by Mark Paluch
parent fae1125e2f
commit a1c139445f
4 changed files with 86 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchOperations.java
View File

@ -20,6 +20,7 @@ import reactor.core.publisher.Mono;
import org.elasticsearch.index.query.QueryBuilders;
import org.reactivestreams.Publisher;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient;
import org.springframework.data.elasticsearch.core.query.Query;
import org.springframework.data.elasticsearch.core.query.StringQuery;
@ -205,7 +206,10 @@ public interface ReactiveElasticsearchOperations {
Mono<Boolean> exists(String id, Class<?> entityType, @Nullable String index, @Nullable String type);
/**
* Search the index for entities matching the given {@link Query query}.
* Search the index for entities matching the given {@link Query query}. <br />
* {@link Pageable#isUnpaged() Unpaged} queries may overrule elasticsearch server defaults for page size by either
* delegating to the scroll API or using a max {@link org.elasticsearch.search.builder.SearchSourceBuilder#size(int)
* size}.
*
* @param query must not be {@literal null}.
* @param entityType must not be {@literal null}.
@ -217,7 +221,10 @@ public interface ReactiveElasticsearchOperations {
}
/**
* Search the index for entities matching the given {@link Query query}.
* Search the index for entities matching the given {@link Query query}. <br />
* {@link Pageable#isUnpaged() Unpaged} queries may overrule elasticsearch server defaults for page size by either *
* delegating to the scroll API or using a max {@link org.elasticsearch.search.builder.SearchSourceBuilder#size(int) *
* size}.
*
* @param query must not be {@literal null}.
* @param entityType The entity type for mapping the query. Must not be {@literal null}.

47
src/main/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchTemplate.java
View File

@ -253,6 +253,16 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
searchSourceBuilder.fetchSource(query.getSourceFilter().getIncludes(), query.getSourceFilter().getExcludes());
}
sort(query, entity).forEach(searchSourceBuilder::sort);
if (query.getMinScore() > 0) {
searchSourceBuilder.minScore(query.getMinScore());
}
if (query.getIndicesOptions() != null) {
request.indicesOptions(query.getIndicesOptions());
}
if (query.getPageable().isPaged()) {
long offset = query.getPageable().getOffset();
@ -262,25 +272,15 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
searchSourceBuilder.from((int) offset);
searchSourceBuilder.size(query.getPageable().getPageSize());
request.source(searchSourceBuilder);
return doFind(prepareSearchRequest(request));
} else {
searchSourceBuilder.from(0);
searchSourceBuilder.size(10000); // this is the index.max_result_window default value
request.source(searchSourceBuilder);
return doScan(prepareSearchRequest(request));
}
if (query.getIndicesOptions() != null) {
request.indicesOptions(query.getIndicesOptions());
}
sort(query, entity).forEach(searchSourceBuilder::sort);
if (query.getMinScore() > 0) {
searchSourceBuilder.minScore(query.getMinScore());
}
request.source(searchSourceBuilder);
return doFind(prepareSearchRequest(request));
});
}
@ -516,6 +516,21 @@ public class ReactiveElasticsearchTemplate implements ReactiveElasticsearchOpera
return Flux.from(execute(client -> client.search(request)));
}
/**
* Customization hook on the actual execution result {@link Publisher}. <br />
*
* @param request the already prepared {@link SearchRequest} ready to be executed.
* @return a {@link Flux} emitting the result of the operation.
*/
protected Flux<SearchHit> doScan(SearchRequest request) {
if (QUERY_LOGGER.isDebugEnabled()) {
QUERY_LOGGER.debug("Executing doScan: {}", request);
}
return Flux.from(execute(client -> client.scroll(request)));
}
/**
* Customization hook on the actual execution result {@link Publisher}. <br />
*

38
src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchTemplateTests.java
View File

@ -32,6 +32,7 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.junit.Before;
import org.junit.Rule;
@ -39,6 +40,9 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.dao.DataAccessResourceFailureException;
import org.springframework.data.annotation.Id;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.data.elasticsearch.ElasticsearchVersion;
import org.springframework.data.elasticsearch.ElasticsearchVersionRule;
import org.springframework.data.elasticsearch.TestUtils;
@ -354,6 +358,40 @@ public class ReactiveElasticsearchTemplateTests {
.verifyComplete();
}
@Test // DATAES-518
public void findShouldApplyPagingCorrectly() {
List<SampleEntity> source = IntStream.range(0, 100).mapToObj(it -> randomEntity("entity - " + it))
.collect(Collectors.toList());
index(source.toArray(new SampleEntity[0]));
CriteriaQuery query = new CriteriaQuery(new Criteria("message").contains("entity")) //
.addSort(Sort.by("message"))//
.setPageable(PageRequest.of(0, 20));
template.find(query, SampleEntity.class).as(StepVerifier::create) //
.expectNextCount(20) //
.verifyComplete();
}
@Test // DATAES-518
public void findWithoutPagingShouldReadAll() {
List<SampleEntity> source = IntStream.range(0, 100).mapToObj(it -> randomEntity("entity - " + it))
.collect(Collectors.toList());
index(source.toArray(new SampleEntity[0]));
CriteriaQuery query = new CriteriaQuery(new Criteria("message").contains("entity")) //
.addSort(Sort.by("message"))//
.setPageable(Pageable.unpaged());
template.find(query, SampleEntity.class).as(StepVerifier::create) //
.expectNextCount(100) //
.verifyComplete();
}
@Test // DATAES-504
public void countShouldReturnCountAllWhenGivenNoQuery() {

17
src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchTemplateUnitTests.java
View File

@ -92,20 +92,20 @@ public class ReactiveElasticsearchTemplateUnitTests {
assertThat(captor.getValue().getRefreshPolicy()).isEqualTo(RefreshPolicy.WAIT_UNTIL);
}
@Test // DATAES-504
@Test // DATAES-504, DATAES-518
public void findShouldFallBackToDefaultIndexOptionsIfNotSet() {
ArgumentCaptor<SearchRequest> captor = ArgumentCaptor.forClass(SearchRequest.class);
when(client.search(captor.capture())).thenReturn(Flux.empty());
template.find(new CriteriaQuery(new Criteria("*")), SampleEntity.class) //
template.find(new CriteriaQuery(new Criteria("*")).setPageable(PageRequest.of(0, 10)), SampleEntity.class) //
.as(StepVerifier::create) //
.verifyComplete();
assertThat(captor.getValue().indicesOptions()).isEqualTo(DEFAULT_INDICES_OPTIONS);
}
@Test // DATAES-504
@Test // DATAES-504, DATAES-518
public void findShouldApplyIndexOptionsIfSet() {
ArgumentCaptor<SearchRequest> captor = ArgumentCaptor.forClass(SearchRequest.class);
@ -113,7 +113,7 @@ public class ReactiveElasticsearchTemplateUnitTests {
template.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN);
template.find(new CriteriaQuery(new Criteria("*")), SampleEntity.class) //
template.find(new CriteriaQuery(new Criteria("*")).setPageable(PageRequest.of(0, 10)), SampleEntity.class) //
.as(StepVerifier::create) //
.verifyComplete();
@ -135,19 +135,18 @@ public class ReactiveElasticsearchTemplateUnitTests {
assertThat(captor.getValue().source().size()).isEqualTo(50);
}
@Test // DATAES-504
public void findShouldApplyDefaultMaxIfPaginationNotSet() {
@Test // DATAES-504, DATAES-518
public void findShouldUseScrollIfPaginationNotSet() {
ArgumentCaptor<SearchRequest> captor = ArgumentCaptor.forClass(SearchRequest.class);
when(client.search(captor.capture())).thenReturn(Flux.empty());
when(client.scroll(captor.capture())).thenReturn(Flux.empty());
template.find(new CriteriaQuery(new Criteria("*")).setPageable(Pageable.unpaged()), SampleEntity.class) //
.as(StepVerifier::create) //
.verifyComplete();
assertThat(captor.getValue().source().from()).isEqualTo(0);
assertThat(captor.getValue().source().size()).isEqualTo(10000);
verify(client).scroll(any());
}
@Test // DATAES-504