From 23e1e9f46a179022334dbee6ebddb64e4488f1ab Mon Sep 17 00:00:00 2001 From: Kevin Leturc Date: Sun, 6 Dec 2015 20:34:36 +0100 Subject: [PATCH] DATAES-217 Propose scan method with a class in order to retrieve index and type from Document --- .../core/ElasticsearchOperations.java | 48 +++++++++-- .../core/ElasticsearchTemplate.java | 80 ++++++++++++------- .../core/ElasticsearchTemplateTests.java | 58 ++++++++++++++ 3 files changed, 147 insertions(+), 39 deletions(-) diff --git a/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchOperations.java b/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchOperations.java index a0ad2c30c..53ac150d2 100755 --- a/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchOperations.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchOperations.java @@ -484,23 +484,55 @@ public interface ElasticsearchOperations { /** * Returns scroll id for criteria query * - * @param query - * @param scrollTimeInMillis - * @param noFields - * @return + * @param query The criteria query. + * @param scrollTimeInMillis The time in millisecond for scroll feature + * {@link org.elasticsearch.action.search.SearchRequestBuilder#setScroll(org.elasticsearch.common.unit.TimeValue)}. + * @param noFields The no fields support + * {@link org.elasticsearch.action.search.SearchRequestBuilder#setNoFields()}. + * @return The scan id for input query. */ String scan(CriteriaQuery query, long scrollTimeInMillis, boolean noFields); + /** + * Returns scroll id for criteria query + * + * @param query The criteria query. + * @param scrollTimeInMillis The time in millisecond for scroll feature + * {@link org.elasticsearch.action.search.SearchRequestBuilder#setScroll(org.elasticsearch.common.unit.TimeValue)}. + * @param noFields The no fields support + * {@link org.elasticsearch.action.search.SearchRequestBuilder#setNoFields()}. + * @param clazz The class of entity to retrieve. + * @param The type of entity to retrieve. + * @return The scan id for input query. + */ + String scan(CriteriaQuery query, long scrollTimeInMillis, boolean noFields, Class clazz); + /** * Returns scroll id for scan query * - * @param query - * @param scrollTimeInMillis - * @param noFields - * @return + * @param query The search query. + * @param scrollTimeInMillis The time in millisecond for scroll feature + * {@link org.elasticsearch.action.search.SearchRequestBuilder#setScroll(org.elasticsearch.common.unit.TimeValue)}. + * @param noFields The no fields support + * {@link org.elasticsearch.action.search.SearchRequestBuilder#setNoFields()}. + * @return The scan id for input query. */ String scan(SearchQuery query, long scrollTimeInMillis, boolean noFields); + /** + * Returns scroll id for scan query + * + * @param query The search query. + * @param scrollTimeInMillis The time in millisecond for scroll feature + * {@link org.elasticsearch.action.search.SearchRequestBuilder#setScroll(org.elasticsearch.common.unit.TimeValue)}. + * @param noFields The no fields support + * {@link org.elasticsearch.action.search.SearchRequestBuilder#setNoFields()}. + * @param clazz The class of entity to retrieve. + * @param The type of entity to retrieve. + * @return The scan id for input query. + */ + String scan(SearchQuery query, long scrollTimeInMillis, boolean noFields, Class clazz); + /** * Scrolls the results for give scroll id * diff --git a/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplate.java index 2b9c0050d..cd5288100 100755 --- a/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplate.java @@ -337,8 +337,7 @@ public class ElasticsearchTemplate implements ElasticsearchOperations, Applicati @Override public CloseableIterator stream(CriteriaQuery query, Class clazz) { final long scrollTimeInMillis = TimeValue.timeValueMinutes(1).millis(); - setPersistentEntityIndexAndType(query, clazz); - final String initScrollId = scan(query, scrollTimeInMillis, false); + final String initScrollId = scan(query, scrollTimeInMillis, false, clazz); return doStream(initScrollId, scrollTimeInMillis, clazz, resultsMapper); } @@ -350,8 +349,7 @@ public class ElasticsearchTemplate implements ElasticsearchOperations, Applicati @Override public CloseableIterator stream(SearchQuery query, final Class clazz, final SearchResultMapper mapper) { final long scrollTimeInMillis = TimeValue.timeValueMinutes(1).millis(); - setPersistentEntityIndexAndType(query, clazz); - final String initScrollId = scan(query, scrollTimeInMillis, false); + final String initScrollId = scan(query, scrollTimeInMillis, false, clazz); return doStream(initScrollId, scrollTimeInMillis, clazz, mapper); } @@ -675,40 +673,27 @@ public class ElasticsearchTemplate implements ElasticsearchOperations, Applicati @Override public String scan(CriteriaQuery criteriaQuery, long scrollTimeInMillis, boolean noFields) { - Assert.notNull(criteriaQuery.getIndices(), "No index defined for Query"); - Assert.notNull(criteriaQuery.getTypes(), "No type define for Query"); - Assert.notNull(criteriaQuery.getPageable(), "Query.pageable is required for scan & scroll"); + return doScan(prepareScan(criteriaQuery, scrollTimeInMillis, noFields), criteriaQuery); + } - QueryBuilder elasticsearchQuery = new CriteriaQueryProcessor().createQueryFromCriteria(criteriaQuery.getCriteria()); - FilterBuilder elasticsearchFilter = new CriteriaFilterProcessor().createFilterFromCriteria(criteriaQuery.getCriteria()); - SearchRequestBuilder requestBuilder = prepareScan(criteriaQuery, scrollTimeInMillis, noFields); - - if (elasticsearchQuery != null) { - requestBuilder.setQuery(elasticsearchQuery); - } else { - requestBuilder.setQuery(QueryBuilders.matchAllQuery()); - } - - if (elasticsearchFilter != null) { - requestBuilder.setPostFilter(elasticsearchFilter); - } - - return getSearchResponse(requestBuilder.execute()).getScrollId(); + @Override + public String scan(CriteriaQuery criteriaQuery, long scrollTimeInMillis, boolean noFields, Class clazz) { + return doScan(prepareScan(criteriaQuery, scrollTimeInMillis, noFields, clazz), criteriaQuery); } @Override public String scan(SearchQuery searchQuery, long scrollTimeInMillis, boolean noFields) { - Assert.notNull(searchQuery.getIndices(), "No index defined for Query"); - Assert.notNull(searchQuery.getTypes(), "No type define for Query"); - Assert.notNull(searchQuery.getPageable(), "Query.pageable is required for scan & scroll"); + return doScan(prepareScan(searchQuery, scrollTimeInMillis, noFields), searchQuery); + } - SearchRequestBuilder requestBuilder = prepareScan(searchQuery, scrollTimeInMillis, noFields); + @Override + public String scan(SearchQuery searchQuery, long scrollTimeInMillis, boolean noFields, Class clazz) { + return doScan(prepareScan(searchQuery, scrollTimeInMillis, noFields, clazz), searchQuery); + } - if (searchQuery.getFilter() != null) { - requestBuilder.setPostFilter(searchQuery.getFilter()); - } - - return getSearchResponse(requestBuilder.setQuery(searchQuery.getQuery()).execute()).getScrollId(); + private SearchRequestBuilder prepareScan(Query query, long scrollTimeInMillis, boolean noFields, Class clazz) { + setPersistentEntityIndexAndType(query, clazz); + return prepareScan(query, scrollTimeInMillis, noFields); } private SearchRequestBuilder prepareScan(Query query, long scrollTimeInMillis, boolean noFields) { @@ -727,6 +712,39 @@ public class ElasticsearchTemplate implements ElasticsearchOperations, Applicati return requestBuilder; } + private String doScan(SearchRequestBuilder requestBuilder, CriteriaQuery criteriaQuery) { + Assert.notNull(criteriaQuery.getIndices(), "No index defined for Query"); + Assert.notNull(criteriaQuery.getTypes(), "No type define for Query"); + Assert.notNull(criteriaQuery.getPageable(), "Query.pageable is required for scan & scroll"); + + QueryBuilder elasticsearchQuery = new CriteriaQueryProcessor().createQueryFromCriteria(criteriaQuery.getCriteria()); + FilterBuilder elasticsearchFilter = new CriteriaFilterProcessor().createFilterFromCriteria(criteriaQuery.getCriteria()); + + if (elasticsearchQuery != null) { + requestBuilder.setQuery(elasticsearchQuery); + } else { + requestBuilder.setQuery(QueryBuilders.matchAllQuery()); + } + + if (elasticsearchFilter != null) { + requestBuilder.setPostFilter(elasticsearchFilter); + } + + return getSearchResponse(requestBuilder.execute()).getScrollId(); + } + + private String doScan(SearchRequestBuilder requestBuilder, SearchQuery searchQuery) { + Assert.notNull(searchQuery.getIndices(), "No index defined for Query"); + Assert.notNull(searchQuery.getTypes(), "No type define for Query"); + Assert.notNull(searchQuery.getPageable(), "Query.pageable is required for scan & scroll"); + + if (searchQuery.getFilter() != null) { + requestBuilder.setPostFilter(searchQuery.getFilter()); + } + + return getSearchResponse(requestBuilder.setQuery(searchQuery.getQuery()).execute()).getScrollId(); + } + @Override public Page scroll(String scrollId, long scrollTimeInMillis, Class clazz) { SearchResponse response = getSearchResponse(client.prepareSearchScroll(scrollId) diff --git a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplateTests.java b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplateTests.java index f5dcebe2b..492518fe5 100755 --- a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplateTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplateTests.java @@ -900,6 +900,64 @@ public class ElasticsearchTemplateTests { assertThat(sampleEntities.size(), is(equalTo(30))); } + /* + DATAES-217 + */ + @Test + public void shouldReturnResultsWithScanAndScrollForGivenCriteriaQueryAndClass() { + //given + List entities = createSampleEntitiesWithMessage("Test message", 30); + // when + elasticsearchTemplate.bulkIndex(entities); + elasticsearchTemplate.refresh(SampleEntity.class, true); + // then + + CriteriaQuery criteriaQuery = new CriteriaQuery(new Criteria()); + criteriaQuery.setPageable(new PageRequest(0, 10)); + + String scrollId = elasticsearchTemplate.scan(criteriaQuery, 1000, false, SampleEntity.class); + List sampleEntities = new ArrayList(); + boolean hasRecords = true; + while (hasRecords) { + Page page = elasticsearchTemplate.scroll(scrollId, 5000L, SampleEntity.class); + if (page.hasContent()) { + sampleEntities.addAll(page.getContent()); + } else { + hasRecords = false; + } + } + assertThat(sampleEntities.size(), is(equalTo(30))); + } + + /* + DATAES-217 + */ + @Test + public void shouldReturnResultsWithScanAndScrollForGivenSearchQueryAndClass() { + //given + List entities = createSampleEntitiesWithMessage("Test message", 30); + // when + elasticsearchTemplate.bulkIndex(entities); + elasticsearchTemplate.refresh(SampleEntity.class, true); + // then + + SearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery()) + .withPageable(new PageRequest(0, 10)).build(); + + String scrollId = elasticsearchTemplate.scan(searchQuery, 1000, false, SampleEntity.class); + List sampleEntities = new ArrayList(); + boolean hasRecords = true; + while (hasRecords) { + Page page = elasticsearchTemplate.scroll(scrollId, 5000L, SampleEntity.class); + if (page.hasContent()) { + sampleEntities.addAll(page.getContent()); + } else { + hasRecords = false; + } + } + assertThat(sampleEntities.size(), is(equalTo(30))); + } + /* DATAES-167 */