From 5ca0ed7ff120994061b35f165aa5540ee19a3d9d Mon Sep 17 00:00:00 2001 From: Mohsin Husen Date: Wed, 23 Apr 2014 12:29:32 +0100 Subject: [PATCH] DATAES-84 - Add field criteria with searchQuery not supported in scan and scroll - added support for fields in scan and scroll - fixed build failure, separated aliasTests --- .../core/ElasticsearchTemplate.java | 4 + .../data/elasticsearch/core/AliasTests.java | 203 ++++++++++++++++++ .../core/ElasticsearchTemplateTests.java | 187 +++++----------- .../elasticsearch/entities/SampleEntity.java | 2 +- 4 files changed, 259 insertions(+), 137 deletions(-) create mode 100644 src/test/java/org/springframework/data/elasticsearch/core/AliasTests.java diff --git a/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplate.java index a13b997c0..b57116642 100755 --- a/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplate.java @@ -443,6 +443,10 @@ public class ElasticsearchTemplate implements ElasticsearchOperations { requestBuilder.setPostFilter(searchQuery.getFilter()); } + if(isNotEmpty(searchQuery.getFields())) { + requestBuilder.addFields(toArray(searchQuery.getFields())); + } + if (noFields) { requestBuilder.setNoFields(); } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/AliasTests.java b/src/test/java/org/springframework/data/elasticsearch/core/AliasTests.java new file mode 100644 index 000000000..39b54aa60 --- /dev/null +++ b/src/test/java/org/springframework/data/elasticsearch/core/AliasTests.java @@ -0,0 +1,203 @@ +/* +* Copyright 2014 the original author or authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.springframework.data.elasticsearch.core; + +import static org.apache.commons.lang.RandomStringUtils.*; +import static org.elasticsearch.index.query.QueryBuilders.*; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.*; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.elasticsearch.builder.SampleEntityBuilder; +import org.springframework.data.elasticsearch.core.query.*; +import org.springframework.data.elasticsearch.entities.SampleEntity; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +/** + * @author Mohsin Husen + */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration("classpath:elasticsearch-template-test.xml") +public class AliasTests { + + private static final String INDEX_NAME = "test-alias-index"; + private static final String TYPE_NAME = "test-alias-type"; + + @Autowired + private ElasticsearchTemplate elasticsearchTemplate; + + @Before + public void before() { + Map settings = new HashMap(); + settings.put("index.refresh_interval", "-1"); + settings.put("index.number_of_replicas", "0"); + settings.put("index.number_of_shards", "2"); + settings.put("index.store.type", "memory"); + + elasticsearchTemplate.deleteIndex(INDEX_NAME); + elasticsearchTemplate.createIndex(INDEX_NAME, settings); + elasticsearchTemplate.refresh(INDEX_NAME, true); + } + + @Test + public void shouldAddAlias() { + // given + String aliasName = "test-alias"; + AliasQuery aliasQuery = new AliasBuilder() + .withIndexName(INDEX_NAME) + .withAliasName(aliasName).build(); + // when + elasticsearchTemplate.addAlias(aliasQuery); + // then + Set aliases = elasticsearchTemplate.queryForAlias(INDEX_NAME); + assertThat(aliases, is(notNullValue())); + assertThat(aliases.contains(aliasName), is(true)); + } + + @Test + public void shouldRemoveAlias() { + // given + String indexName = INDEX_NAME; + String aliasName = "test-alias"; + AliasQuery aliasQuery = new AliasBuilder() + .withIndexName(indexName) + .withAliasName(aliasName).build(); + // when + elasticsearchTemplate.addAlias(aliasQuery); + Set aliases = elasticsearchTemplate.queryForAlias(indexName); + assertThat(aliases, is(notNullValue())); + assertThat(aliases.contains(aliasName), is(true)); + // then + elasticsearchTemplate.removeAlias(aliasQuery); + aliases = elasticsearchTemplate.queryForAlias(indexName); + assertThat(aliases, is(notNullValue())); + assertThat(aliases.size(), is(0)); + } + + /* + DATAES-70 + */ + @Test + public void shouldAddAliasWithGivenRoutingValue() { + //given + String indexName = INDEX_NAME; + String alias = "test-alias"; + + AliasQuery aliasQuery = new AliasBuilder() + .withIndexName(indexName) + .withAliasName(alias) + .withRouting("0").build(); + + //when + elasticsearchTemplate.addAlias(aliasQuery); + + String documentId = randomNumeric(5); + SampleEntity sampleEntity = new SampleEntityBuilder(documentId) + .message("some message") + .version(System.currentTimeMillis()).build(); + + IndexQuery indexQuery = new IndexQueryBuilder() + .withIndexName(alias) + .withId(sampleEntity.getId()) + .withType(TYPE_NAME) + .withObject(sampleEntity) + .build(); + + elasticsearchTemplate.index(indexQuery); + elasticsearchTemplate.refresh(INDEX_NAME, true); + + SearchQuery query = new NativeSearchQueryBuilder().withQuery(matchAllQuery()) + .withIndices(alias).withTypes(TYPE_NAME).build(); + long count = elasticsearchTemplate.count(query); + //then + Set aliases = elasticsearchTemplate.queryForAlias(INDEX_NAME); + assertThat(aliases, is(notNullValue())); + assertThat(aliases.contains(alias), is(true)); + assertThat(count, is(1L)); + + //cleanup + elasticsearchTemplate.removeAlias(aliasQuery); + elasticsearchTemplate.deleteIndex(SampleEntity.class); + elasticsearchTemplate.createIndex(SampleEntity.class); + elasticsearchTemplate.putMapping(SampleEntity.class); + elasticsearchTemplate.refresh(SampleEntity.class, true); + } + + /* + DATAES-70 + */ + @Test + public void shouldAddAliasForVariousRoutingValues() { + //given + String alias1 = "test-alias-1"; + String alias2 = "test-alias-2"; + + AliasQuery aliasQuery1 = new AliasBuilder() + .withIndexName(INDEX_NAME) + .withAliasName(alias1) + .withIndexRouting("0").build(); + + AliasQuery aliasQuery2 = new AliasBuilder() + .withIndexName(INDEX_NAME) + .withAliasName(alias2) + .withSearchRouting("1").build(); + + //when + elasticsearchTemplate.addAlias(aliasQuery1); + elasticsearchTemplate.addAlias(aliasQuery2); + + String documentId = randomNumeric(5); + SampleEntity sampleEntity = new SampleEntityBuilder(documentId) + .message("some message") + .version(System.currentTimeMillis()).build(); + + IndexQuery indexQuery = new IndexQueryBuilder() + .withIndexName(alias1) + .withType(TYPE_NAME) + .withId(sampleEntity.getId()) + .withObject(sampleEntity).build(); + + elasticsearchTemplate.index(indexQuery); + elasticsearchTemplate.refresh(SampleEntity.class, true); + + SearchQuery query = new NativeSearchQueryBuilder().withQuery(matchAllQuery()) + .withIndices(alias2).withTypes(TYPE_NAME).build(); + long count = elasticsearchTemplate.count(query, SampleEntity.class); + // then + Set aliases = elasticsearchTemplate.queryForAlias(INDEX_NAME); + assertThat(aliases, is(notNullValue())); + assertThat(aliases.contains(alias1), is(true)); + assertThat(aliases.contains(alias2), is(true)); + assertThat(count, is(0L)); + + //cleanup + elasticsearchTemplate.removeAlias(aliasQuery1); + elasticsearchTemplate.removeAlias(aliasQuery2); + elasticsearchTemplate.deleteIndex(SampleEntity.class); + elasticsearchTemplate.createIndex(SampleEntity.class); + elasticsearchTemplate.putMapping(SampleEntity.class); + elasticsearchTemplate.refresh(SampleEntity.class, true); + } + +} diff --git a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplateTests.java b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplateTests.java index 8ad82c9b4..e59b4d415 100755 --- a/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplateTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplateTests.java @@ -594,6 +594,56 @@ public class ElasticsearchTemplateTests { assertThat(sampleEntities.size(), is(equalTo(30))); } + /* + DATAES-84 + */ + @Test + public void shouldReturnResultsWithScanAndScrollForSpecifiedFields() { + //given + List entities = createSampleEntitiesWithMessage("Test message", 30); + // when + elasticsearchTemplate.bulkIndex(entities); + elasticsearchTemplate.refresh(SampleEntity.class, true); + // then + + SearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery()) + .withIndices(INDEX_NAME) + .withTypes(TYPE_NAME) + .withFields("message") + .withPageable(new PageRequest(0, 10)) + .build(); + + String scrollId = elasticsearchTemplate.scan(searchQuery, 5000, false); + List sampleEntities = new ArrayList(); + boolean hasRecords = true; + while (hasRecords) { + Page page = elasticsearchTemplate.scroll(scrollId, 5000L, new SearchResultMapper() { + @Override + public FacetedPage mapResults(SearchResponse response, Class clazz, Pageable pageable) { + List result = new ArrayList(); + for(SearchHit searchHit : response.getHits()){ + String message = searchHit.getFields().get("message").getValue(); + SampleEntity sampleEntity = new SampleEntity(); + sampleEntity.setId(searchHit.getId()); + sampleEntity.setMessage(message); + result.add(sampleEntity); + } + + if(result.size() > 0) { + return new FacetedPageImpl((List) result); + } + return null; + } + }); + if (page != null) { + sampleEntities.addAll(page.getContent()); + } else { + hasRecords = false; + } + } + assertThat(sampleEntities.size(), is(equalTo(30))); + } + @Test public void shouldReturnResultsForScanAndScrollWithCustomResultMapper() { //given @@ -898,141 +948,6 @@ public class ElasticsearchTemplateTests { assertThat(sampleEntities.getTotalElements(), equalTo(0L)); } - @Test - public void shouldAddAlias() { - // given - elasticsearchTemplate.createIndex(SampleEntity.class); - String aliasName = "test-alias"; - AliasQuery aliasQuery = new AliasBuilder() - .withIndexName(INDEX_NAME) - .withAliasName(aliasName).build(); - // when - elasticsearchTemplate.addAlias(aliasQuery); - // then - Set aliases = elasticsearchTemplate.queryForAlias(INDEX_NAME); - assertThat(aliases, is(notNullValue())); - assertThat(aliases.contains(aliasName), is(true)); - } - - @Test - public void shouldRemoveAlias() { - // given - elasticsearchTemplate.createIndex(SampleEntity.class); - String indexName = INDEX_NAME; - String aliasName = "test-alias"; - AliasQuery aliasQuery = new AliasBuilder() - .withIndexName(indexName) - .withAliasName(aliasName).build(); - // when - elasticsearchTemplate.addAlias(aliasQuery); - Set aliases = elasticsearchTemplate.queryForAlias(indexName); - assertThat(aliases, is(notNullValue())); - assertThat(aliases.contains(aliasName), is(true)); - // then - elasticsearchTemplate.removeAlias(aliasQuery); - aliases = elasticsearchTemplate.queryForAlias(indexName); - assertThat(aliases, is(notNullValue())); - assertThat(aliases.size(), is(0)); - } - - /* - DATAES-70 - */ - @Test - public void shouldAddAliasWithGivenRoutingValue() { - //given - String indexName = INDEX_NAME; - String alias = "test-alias"; - - AliasQuery aliasQuery = new AliasBuilder() - .withIndexName(indexName) - .withAliasName(alias) - .withRouting("0").build(); - - //when - elasticsearchTemplate.addAlias(aliasQuery); - - String documentId = randomNumeric(5); - SampleEntity sampleEntity = new SampleEntityBuilder(documentId) - .message("some message") - .version(System.currentTimeMillis()).build(); - - IndexQuery indexQuery = new IndexQueryBuilder() - .withIndexName(alias) - .withId(sampleEntity.getId()) - .withObject(sampleEntity).build(); - - elasticsearchTemplate.index(indexQuery); - elasticsearchTemplate.refresh(SampleEntity.class, true); - - SearchQuery query = new NativeSearchQueryBuilder().withQuery(matchAllQuery()) - .withIndices(alias).build(); - long count = elasticsearchTemplate.count(query, SampleEntity.class); - //then - Set aliases = elasticsearchTemplate.queryForAlias(INDEX_NAME); - assertThat(aliases, is(notNullValue())); - assertThat(aliases.contains(alias), is(true)); - assertThat(count, is(1L)); - - //cleanup - elasticsearchTemplate.removeAlias(aliasQuery); - elasticsearchTemplate.refresh(SampleEntity.class, true); - } - - /* - DATAES-70 - */ - @Test - public void shouldAddAliasForVariousRoutingValues() { - //given - String indexName = INDEX_NAME; - String alias1 = "test-alias-1"; - String alias2 = "test-alias-2"; - - AliasQuery aliasQuery1 = new AliasBuilder() - .withIndexName(indexName) - .withAliasName(alias1) - .withIndexRouting("0").build(); - - AliasQuery aliasQuery2 = new AliasBuilder() - .withIndexName(indexName) - .withAliasName(alias2) - .withSearchRouting("1").build(); - - //when - elasticsearchTemplate.addAlias(aliasQuery1); - elasticsearchTemplate.addAlias(aliasQuery2); - - String documentId = randomNumeric(5); - SampleEntity sampleEntity = new SampleEntityBuilder(documentId) - .message("some message") - .version(System.currentTimeMillis()).build(); - - IndexQuery indexQuery = new IndexQueryBuilder() - .withIndexName(alias1) - .withId(sampleEntity.getId()) - .withObject(sampleEntity).build(); - - elasticsearchTemplate.index(indexQuery); - elasticsearchTemplate.refresh(SampleEntity.class, true); - - SearchQuery query = new NativeSearchQueryBuilder().withQuery(matchAllQuery()) - .withIndices(alias2).build(); - long count = elasticsearchTemplate.count(query, SampleEntity.class); - // then - Set aliases = elasticsearchTemplate.queryForAlias(INDEX_NAME); - assertThat(aliases, is(notNullValue())); - assertThat(aliases.contains(alias1), is(true)); - assertThat(aliases.contains(alias2), is(true)); - assertThat(count, is(0L)); - - //cleanup - elasticsearchTemplate.removeAlias(aliasQuery1); - elasticsearchTemplate.removeAlias(aliasQuery2); - elasticsearchTemplate.refresh(SampleEntity.class, true); - } - - @Test public void shouldIndexDocumentForSpecifiedSource() { @@ -1507,7 +1422,7 @@ public class ElasticsearchTemplateTests { assertThat(map.containsKey("index.store.type"), is(true)); assertThat((String) map.get("index.refresh_interval"), is("-1")); assertThat((String) map.get("index.number_of_replicas"), is("0")); - assertThat((String) map.get("index.number_of_shards"), is("2")); + assertThat((String) map.get("index.number_of_shards"), is("1")); assertThat((String) map.get("index.store.type"), is("memory")); } diff --git a/src/test/java/org/springframework/data/elasticsearch/entities/SampleEntity.java b/src/test/java/org/springframework/data/elasticsearch/entities/SampleEntity.java index 72ef060df..52325688f 100644 --- a/src/test/java/org/springframework/data/elasticsearch/entities/SampleEntity.java +++ b/src/test/java/org/springframework/data/elasticsearch/entities/SampleEntity.java @@ -25,7 +25,7 @@ import org.springframework.data.elasticsearch.annotations.Document; * @author Rizwan Idrees * @author Mohsin Husen */ -@Document(indexName = "test-index", type = "test-type", indexStoreType = "memory", shards = 2, replicas = 0, refreshInterval = "-1") +@Document(indexName = "test-index", type = "test-type", indexStoreType = "memory", shards = 1, replicas = 0, refreshInterval = "-1") public class SampleEntity { @Id