Cwikius-Spring/spring-data-elasticsearch
1
0
Fork 0
mirror of https://github.com/spring-projects/spring-data-elasticsearch.git synced 2025-06-13 15:42:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

DATAES-167 - Add scan method for CriteriaQuery.

Browse Source
This commit is contained in:
Kevin Leturc 2015-05-31 17:58:44 +02:00 committed by Artur Konczak
parent c9ffc4417a
commit 576495932e
3 changed files with 178 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchOperations.java
View File

@ -439,6 +439,16 @@ public interface ElasticsearchOperations {
*/
<T> void refresh(Class<T> clazz, boolean waitForOperation);
/**
* Returns scroll id for criteria query
*
* @param query
* @param scrollTimeInMillis
* @param noFields
* @return
*/
String scan(CriteriaQuery query, long scrollTimeInMillis, boolean noFields);
/**
* Returns scroll id for scan query
*

43
src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplate.java
View File

@ -594,29 +594,58 @@ public class ElasticsearchTemplate implements ElasticsearchOperations, Applicati
delete(deleteQuery, clazz);
}
@Override
public String scan(CriteriaQuery criteriaQuery, long scrollTimeInMillis, boolean noFields) {
Assert.notNull(criteriaQuery.getIndices(), "No index defined for Query");
Assert.notNull(criteriaQuery.getTypes(), "No type define for Query");
Assert.notNull(criteriaQuery.getPageable(), "Query.pageable is required for scan & scroll");
QueryBuilder elasticsearchQuery = new CriteriaQueryProcessor().createQueryFromCriteria(criteriaQuery.getCriteria());
FilterBuilder elasticsearchFilter = new CriteriaFilterProcessor().createFilterFromCriteria(criteriaQuery.getCriteria());
SearchRequestBuilder requestBuilder = prepareScan(criteriaQuery, scrollTimeInMillis, noFields);
if (elasticsearchQuery != null) {
requestBuilder.setQuery(elasticsearchQuery);
} else {
requestBuilder.setQuery(QueryBuilders.matchAllQuery());
}
if (elasticsearchFilter != null) {
requestBuilder.setPostFilter(elasticsearchFilter);
}
return getSearchResponse(requestBuilder.execute()).getScrollId();
}
@Override
public String scan(SearchQuery searchQuery, long scrollTimeInMillis, boolean noFields) {
Assert.notNull(searchQuery.getIndices(), "No index defined for Query");
Assert.notNull(searchQuery.getTypes(), "No type define for Query");
Assert.notNull(searchQuery.getPageable(), "Query.pageable is required for scan & scroll");
SearchRequestBuilder requestBuilder = client.prepareSearch(toArray(searchQuery.getIndices())).setSearchType(SCAN)
.setQuery(searchQuery.getQuery()).setTypes(toArray(searchQuery.getTypes()))
.setScroll(TimeValue.timeValueMillis(scrollTimeInMillis)).setFrom(0)
.setSize(searchQuery.getPageable().getPageSize());
SearchRequestBuilder requestBuilder = prepareScan(searchQuery, scrollTimeInMillis, noFields);
if (searchQuery.getFilter() != null) {
requestBuilder.setPostFilter(searchQuery.getFilter());
}
if (isNotEmpty(searchQuery.getFields())) {
requestBuilder.addFields(toArray(searchQuery.getFields()));
return getSearchResponse(requestBuilder.setQuery(searchQuery.getQuery()).execute()).getScrollId();
}
private SearchRequestBuilder prepareScan(Query query, long scrollTimeInMillis, boolean noFields) {
SearchRequestBuilder requestBuilder = client.prepareSearch(toArray(query.getIndices())).setSearchType(SCAN)
.setTypes(toArray(query.getTypes()))
.setScroll(TimeValue.timeValueMillis(scrollTimeInMillis)).setFrom(0)
.setSize(query.getPageable().getPageSize());
if (isNotEmpty(query.getFields())) {
requestBuilder.addFields(toArray(query.getFields()));
}
if (noFields) {
requestBuilder.setNoFields();
}
return getSearchResponse(requestBuilder.execute()).getScrollId();
return requestBuilder;
}
@Override

135
src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplateTests.java
View File

@ -647,8 +647,39 @@ public class ElasticsearchTemplateTests {
assertThat(sampleEntities.getContent(), hasItem(sampleEntity));
}
/*
DATAES-167
*/
@Test
public void shouldReturnResultsWithScanAndScroll() {
public void shouldReturnResultsWithScanAndScrollForGivenCriteriaQuery() {
//given
List<IndexQuery> entities = createSampleEntitiesWithMessage("Test message", 30);
// when
elasticsearchTemplate.bulkIndex(entities);
elasticsearchTemplate.refresh(SampleEntity.class, true);
// then
CriteriaQuery criteriaQuery = new CriteriaQuery(new Criteria());
criteriaQuery.addIndices(INDEX_NAME);
criteriaQuery.addTypes(TYPE_NAME);
criteriaQuery.setPageable(new PageRequest(0, 10));
String scrollId = elasticsearchTemplate.scan(criteriaQuery, 1000, false);
List<SampleEntity> sampleEntities = new ArrayList<SampleEntity>();
boolean hasRecords = true;
while (hasRecords) {
Page<SampleEntity> page = elasticsearchTemplate.scroll(scrollId, 5000L, SampleEntity.class);
if (page.hasContent()) {
sampleEntities.addAll(page.getContent());
} else {
hasRecords = false;
}
}
assertThat(sampleEntities.size(), is(equalTo(30)));
}
@Test
public void shouldReturnResultsWithScanAndScrollForGivenSearchQuery() {
//given
List<IndexQuery> entities = createSampleEntitiesWithMessage("Test message", 30);
// when
@ -673,11 +704,60 @@ public class ElasticsearchTemplateTests {
assertThat(sampleEntities.size(), is(equalTo(30)));
}
/*
DATAES-167
*/
@Test
public void shouldReturnResultsWithScanAndScrollForSpecifiedFieldsForCriteriaCriteria() {
//given
List<IndexQuery> entities = createSampleEntitiesWithMessage("Test message", 30);
// when
elasticsearchTemplate.bulkIndex(entities);
elasticsearchTemplate.refresh(SampleEntity.class, true);
// then
CriteriaQuery criteriaQuery = new CriteriaQuery(new Criteria());
criteriaQuery.addIndices(INDEX_NAME);
criteriaQuery.addTypes(TYPE_NAME);
criteriaQuery.addFields("message");
criteriaQuery.setPageable(new PageRequest(0, 10));
String scrollId = elasticsearchTemplate.scan(criteriaQuery, 5000, false);
List<SampleEntity> sampleEntities = new ArrayList<SampleEntity>();
boolean hasRecords = true;
while (hasRecords) {
Page<SampleEntity> page = elasticsearchTemplate.scroll(scrollId, 5000L, new SearchResultMapper() {
@Override
public <T> FacetedPage<T> mapResults(SearchResponse response, Class<T> clazz, Pageable pageable) {
List<SampleEntity> result = new ArrayList<SampleEntity>();
for (SearchHit searchHit : response.getHits()) {
String message = searchHit.getFields().get("message").getValue();
SampleEntity sampleEntity = new SampleEntity();
sampleEntity.setId(searchHit.getId());
sampleEntity.setMessage(message);
result.add(sampleEntity);
}
if (result.size() > 0) {
return new FacetedPageImpl<T>((List<T>) result);
}
return null;
}
});
if (page != null) {
sampleEntities.addAll(page.getContent());
} else {
hasRecords = false;
}
}
assertThat(sampleEntities.size(), is(equalTo(30)));
}
/*
DATAES-84
*/
@Test
public void shouldReturnResultsWithScanAndScrollForSpecifiedFields() {
public void shouldReturnResultsWithScanAndScrollForSpecifiedFieldsForSearchCriteria() {
//given
List<IndexQuery> entities = createSampleEntitiesWithMessage("Test message", 30);
// when
@ -723,8 +803,57 @@ public class ElasticsearchTemplateTests {
assertThat(sampleEntities.size(), is(equalTo(30)));
}
/*
DATAES-167
*/
@Test
public void shouldReturnResultsForScanAndScrollWithCustomResultMapper() {
public void shouldReturnResultsForScanAndScrollWithCustomResultMapperForGivenCriteriaQuery() {
//given
List<IndexQuery> entities = createSampleEntitiesWithMessage("Test message", 30);
// when
elasticsearchTemplate.bulkIndex(entities);
elasticsearchTemplate.refresh(SampleEntity.class, true);
// then
CriteriaQuery criteriaQuery = new CriteriaQuery(new Criteria());
criteriaQuery.addIndices(INDEX_NAME);
criteriaQuery.addTypes(TYPE_NAME);
criteriaQuery.setPageable(new PageRequest(0, 10));
String scrollId = elasticsearchTemplate.scan(criteriaQuery, 1000, false);
List<SampleEntity> sampleEntities = new ArrayList<SampleEntity>();
boolean hasRecords = true;
while (hasRecords) {
Page<SampleEntity> page = elasticsearchTemplate.scroll(scrollId, 5000L, new SearchResultMapper() {
@Override
public <T> FacetedPage<T> mapResults(SearchResponse response, Class<T> clazz, Pageable pageable) {
List<SampleEntity> chunk = new ArrayList<SampleEntity>();
for (SearchHit searchHit : response.getHits()) {
if (response.getHits().getHits().length <= 0) {
return null;
}
SampleEntity user = new SampleEntity();
user.setId(searchHit.getId());
user.setMessage((String) searchHit.getSource().get("message"));
chunk.add(user);
}
if (chunk.size() > 0) {
return new FacetedPageImpl<T>((List<T>) chunk);
}
return null;
}
});
if (page != null) {
sampleEntities.addAll(page.getContent());
} else {
hasRecords = false;
}
}
assertThat(sampleEntities.size(), is(equalTo(30)));
}
@Test
public void shouldReturnResultsForScanAndScrollWithCustomResultMapperForGivenSearchQuery() {
//given
List<IndexQuery> entities = createSampleEntitiesWithMessage("Test message", 30);
// when