From 0693923798e3a0e884b104c0d866c5ae1db0b588 Mon Sep 17 00:00:00 2001 From: Peter-Josef Meisch Date: Mon, 6 Jan 2020 22:25:54 +0100 Subject: [PATCH] DATAES-372 - Support highlighting via annotation. Original PR: #377 --- .../elasticsearch/annotations/Highlight.java | 36 ++++ .../annotations/HighlightField.java | 37 ++++ .../annotations/HighlightParameters.java | 78 ++++++++ .../elasticsearch/core/RequestFactory.java | 26 +-- .../data/elasticsearch/core/SearchHit.java | 6 + .../MappingElasticsearchConverter.java | 21 +- .../ElasticsearchPersistentEntity.java | 12 ++ .../SimpleElasticsearchPersistentEntity.java | 26 ++- .../core/query/AbstractQuery.java | 13 ++ .../core/query/HighlightQuery.java | 37 ++++ .../core/query/HighlightQueryBuilder.java | 180 ++++++++++++++++++ .../data/elasticsearch/core/query/Query.java | 17 ++ ...tReactiveElasticsearchRepositoryQuery.java | 4 + .../query/ElasticsearchPartQuery.java | 8 +- .../query/ElasticsearchQueryMethod.java | 35 +++- .../query/ElasticsearchStringQuery.java | 12 +- .../ReactivePartTreeElasticsearchQuery.java | 1 + ...pleElasticsearchPersistentEntityTests.java | 24 ++- .../query/HighlightQueryBuilderTests.java | 141 ++++++++++++++ .../CustomMethodRepositoryBaseTests.java | 35 +++- ...eReactiveElasticsearchRepositoryTests.java | 43 ++++- .../highlights-with-parameters.json | 34 ++++ src/test/resources/highlights/highlights.json | 5 + 23 files changed, 807 insertions(+), 24 deletions(-) create mode 100644 src/main/java/org/springframework/data/elasticsearch/annotations/Highlight.java create mode 100644 src/main/java/org/springframework/data/elasticsearch/annotations/HighlightField.java create mode 100644 src/main/java/org/springframework/data/elasticsearch/annotations/HighlightParameters.java create mode 100644 src/main/java/org/springframework/data/elasticsearch/core/query/HighlightQuery.java create mode 100644 src/main/java/org/springframework/data/elasticsearch/core/query/HighlightQueryBuilder.java create mode 100644 src/test/java/org/springframework/data/elasticsearch/core/query/HighlightQueryBuilderTests.java create mode 100644 src/test/resources/highlights/highlights-with-parameters.json create mode 100644 src/test/resources/highlights/highlights.json diff --git a/src/main/java/org/springframework/data/elasticsearch/annotations/Highlight.java b/src/main/java/org/springframework/data/elasticsearch/annotations/Highlight.java new file mode 100644 index 000000000..3ac6f60ab --- /dev/null +++ b/src/main/java/org/springframework/data/elasticsearch/annotations/Highlight.java @@ -0,0 +1,36 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * @author Peter-Josef Meisch + * @since 4.0 + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +public @interface Highlight { + + HighlightParameters parameters() default @HighlightParameters; + + HighlightField[] fields(); +} diff --git a/src/main/java/org/springframework/data/elasticsearch/annotations/HighlightField.java b/src/main/java/org/springframework/data/elasticsearch/annotations/HighlightField.java new file mode 100644 index 000000000..d65d59d93 --- /dev/null +++ b/src/main/java/org/springframework/data/elasticsearch/annotations/HighlightField.java @@ -0,0 +1,37 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * @author Peter-Josef Meisch + * @since 4.0 + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +public @interface HighlightField { + + /** + * The name of the field to apply highlighting to. This must be the field name of the entity's property, not the name + * of the field in the index mappings. + */ + String name() default ""; + + HighlightParameters parameters() default @HighlightParameters; +} diff --git a/src/main/java/org/springframework/data/elasticsearch/annotations/HighlightParameters.java b/src/main/java/org/springframework/data/elasticsearch/annotations/HighlightParameters.java new file mode 100644 index 000000000..74aad474b --- /dev/null +++ b/src/main/java/org/springframework/data/elasticsearch/annotations/HighlightParameters.java @@ -0,0 +1,78 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * @author Peter-Josef Meisch + * @since 4.0 + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +public @interface HighlightParameters { + String boundaryChars() default ""; + + int boundaryMaxScan() default -1; + + String boundaryScanner() default ""; + + String boundaryScannerLocale() default ""; + + /** + * only used for {@link Highlight}s. + */ + String encoder() default ""; + + boolean forceSource() default false; + + String fragmenter() default ""; + + /** + * only used for {@link HighlightField}s. + */ + int fragmentOffset() default -1; + + int fragmentSize() default -1; + + /** + * only used for {@link HighlightField}s. + */ + String[] matchedFields() default {}; + + int noMatchSize() default -1; + + int numberOfFragments() default -1; + + String order() default ""; + + int phraseLimit() default -1; + + String[] preTags() default {}; + + String[] postTags() default {}; + + boolean requireFieldMatch() default true; + + /** + * only used for {@link Highlight}s. + */ + String tagsSchema() default ""; + + String type() default ""; +} diff --git a/src/main/java/org/springframework/data/elasticsearch/core/RequestFactory.java b/src/main/java/org/springframework/data/elasticsearch/core/RequestFactory.java index 96bf8a49e..172964089 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/RequestFactory.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/RequestFactory.java @@ -241,20 +241,24 @@ class RequestFactory { } public HighlightBuilder highlightBuilder(Query query) { - HighlightBuilder highlightBuilder = null; - if (query instanceof NativeSearchQuery) { - NativeSearchQuery searchQuery = (NativeSearchQuery) query; + HighlightBuilder highlightBuilder = query.getHighlightQuery().map(HighlightQuery::getHighlightBuilder).orElse(null); - if (searchQuery.getHighlightFields() != null || searchQuery.getHighlightBuilder() != null) { - highlightBuilder = searchQuery.getHighlightBuilder(); + if (highlightBuilder == null) { - if (highlightBuilder == null) { - highlightBuilder = new HighlightBuilder(); - } + if (query instanceof NativeSearchQuery) { + NativeSearchQuery searchQuery = (NativeSearchQuery) query; - if (searchQuery.getHighlightFields() != null) { - for (HighlightBuilder.Field highlightField : searchQuery.getHighlightFields()) { - highlightBuilder.field(highlightField); + if (searchQuery.getHighlightFields() != null || searchQuery.getHighlightBuilder() != null) { + highlightBuilder = searchQuery.getHighlightBuilder(); + + if (highlightBuilder == null) { + highlightBuilder = new HighlightBuilder(); + } + + if (searchQuery.getHighlightFields() != null) { + for (HighlightBuilder.Field highlightField : searchQuery.getHighlightFields()) { + highlightBuilder.field(highlightField); + } } } } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/SearchHit.java b/src/main/java/org/springframework/data/elasticsearch/core/SearchHit.java index 7b947030f..c681a8f79 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/SearchHit.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/SearchHit.java @@ -21,6 +21,7 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import org.springframework.lang.Nullable; import org.springframework.util.Assert; @@ -78,6 +79,11 @@ public class SearchHit { return Collections.unmodifiableList(sortValues); } + public Map> getHighlightFields() { + return Collections.unmodifiableMap(highlightFields.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> Collections.unmodifiableList(entry.getValue())))); + } + /** * gets the highlight values for a field. * diff --git a/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java b/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java index 2b204e9a7..9561d95db 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java @@ -184,12 +184,31 @@ public class MappingElasticsearchConverter String id = searchDocument.hasId() ? searchDocument.getId() : null; float score = searchDocument.getScore(); Object[] sortValues = searchDocument.getSortValues(); - Map> highlightFields = searchDocument.getHighlightFields(); + Map> highlightFields = getHighlightsAndRemapFieldNames(type, searchDocument); T content = mapDocument(searchDocument, type); return new SearchHit(id, score, sortValues, highlightFields, content); } + @Nullable + private Map> getHighlightsAndRemapFieldNames(Class type, SearchDocument searchDocument) { + Map> highlightFields = searchDocument.getHighlightFields(); + + if (highlightFields == null) { + return null; + } + + ElasticsearchPersistentEntity persistentEntity = mappingContext.getPersistentEntity(type); + if (persistentEntity == null) { + return highlightFields; + } + + return highlightFields.entrySet().stream().collect(Collectors.toMap(entry -> { + ElasticsearchPersistentProperty property = persistentEntity.getPersistentPropertyWithFieldName(entry.getKey()); + return property != null ? property.getName() : entry.getKey(); + }, Entry::getValue)); + } + @Override @Nullable public T mapDocument(@Nullable Document document, Class type) { diff --git a/src/main/java/org/springframework/data/elasticsearch/core/mapping/ElasticsearchPersistentEntity.java b/src/main/java/org/springframework/data/elasticsearch/core/mapping/ElasticsearchPersistentEntity.java index 6d2ab20cd..8604854d7 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/mapping/ElasticsearchPersistentEntity.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/mapping/ElasticsearchPersistentEntity.java @@ -16,6 +16,7 @@ package org.springframework.data.elasticsearch.core.mapping; import org.elasticsearch.index.VersionType; +import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.mapping.PersistentEntity; import org.springframework.lang.Nullable; @@ -80,4 +81,15 @@ public interface ElasticsearchPersistentEntity extends PersistentEntity extends BasicPersistentEntity implements ElasticsearchPersistentEntity, ApplicationContextAware { @@ -68,6 +73,7 @@ public class SimpleElasticsearchPersistentEntity extends BasicPersistentEntit private @Nullable String settingPath; private VersionType versionType; private boolean createIndexAndMapping; + private final Map fieldNamePropertyCache = new ConcurrentHashMap<>(); public SimpleElasticsearchPersistentEntity(TypeInformation typeInformation) { @@ -233,4 +239,22 @@ public class SimpleElasticsearchPersistentEntity extends BasicPersistentEntit // DATACMNS-1322 switches to proper immutability behavior which Spring Data Elasticsearch // cannot yet implement } + + @Nullable + @Override + public ElasticsearchPersistentProperty getPersistentPropertyWithFieldName(String fieldName) { + + Assert.notNull(fieldName, "fieldName must not be null"); + + return fieldNamePropertyCache.computeIfAbsent(fieldName, key -> { + AtomicReference propertyRef = new AtomicReference<>(); + doWithProperties((PropertyHandler) property -> { + if (key.equals(property.getFieldName())) { + propertyRef.set(property); + } + }); + + return propertyRef.get(); + }); + } } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/query/AbstractQuery.java b/src/main/java/org/springframework/data/elasticsearch/core/query/AbstractQuery.java index 1c2a106d7..8aeba913a 100755 --- a/src/main/java/org/springframework/data/elasticsearch/core/query/AbstractQuery.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/query/AbstractQuery.java @@ -20,6 +20,7 @@ import static java.util.Collections.*; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Optional; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.IndicesOptions; @@ -52,6 +53,7 @@ abstract class AbstractQuery implements Query { protected boolean trackScores; protected String preference; protected Integer maxResults; + protected HighlightQuery highlightQuery; @Override public Sort getSort() { @@ -195,4 +197,15 @@ abstract class AbstractQuery implements Query { public void setMaxResults(Integer maxResults) { this.maxResults = maxResults; } + + @Override + public void setHighlightQuery(HighlightQuery highlightQuery) { + this.highlightQuery = highlightQuery; + } + + @Override + public Optional getHighlightQuery() { + return Optional.ofNullable(highlightQuery); + } + } diff --git a/src/main/java/org/springframework/data/elasticsearch/core/query/HighlightQuery.java b/src/main/java/org/springframework/data/elasticsearch/core/query/HighlightQuery.java new file mode 100644 index 000000000..90396b988 --- /dev/null +++ b/src/main/java/org/springframework/data/elasticsearch/core/query/HighlightQuery.java @@ -0,0 +1,37 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch.core.query; + +import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; + +/** + * Encapsulates an Elasticsearch {@link org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder} to prevent + * leaking of Elasticsearch classes into the query API. + * + * @author Peter-Josef Meisch + * @since 4.0 + */ +public class HighlightQuery { + private final HighlightBuilder highlightBuilder; + + public HighlightQuery(HighlightBuilder highlightBuilder) { + this.highlightBuilder = highlightBuilder; + } + + public HighlightBuilder getHighlightBuilder() { + return highlightBuilder; + } +} diff --git a/src/main/java/org/springframework/data/elasticsearch/core/query/HighlightQueryBuilder.java b/src/main/java/org/springframework/data/elasticsearch/core/query/HighlightQueryBuilder.java new file mode 100644 index 000000000..4f68e6e95 --- /dev/null +++ b/src/main/java/org/springframework/data/elasticsearch/core/query/HighlightQueryBuilder.java @@ -0,0 +1,180 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch.core.query; + +import java.util.Arrays; +import java.util.stream.Collectors; + +import org.elasticsearch.search.fetch.subphase.highlight.AbstractHighlighterBuilder; +import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.springframework.data.elasticsearch.annotations.Highlight; +import org.springframework.data.elasticsearch.annotations.HighlightField; +import org.springframework.data.elasticsearch.annotations.HighlightParameters; +import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentEntity; +import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentProperty; +import org.springframework.data.mapping.context.MappingContext; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; + +/** + * Converts the {@link Highlight} annotation from a method to an Elasticsearch {@link HighlightBuilder}. + * + * @author Peter-Josef Meisch + */ +public class HighlightQueryBuilder { + + private final MappingContext, ElasticsearchPersistentProperty> mappingContext; + + public HighlightQueryBuilder( + MappingContext, ElasticsearchPersistentProperty> mappingContext) { + this.mappingContext = mappingContext; + } + + /** + * creates a HighlightBuilder from an annotation + * + * @param highlight, must not be {@literal null} + * @param type the entity type, used to map field names. If null, field names are not mapped. + * @return the builder for the highlight + */ + public HighlightQuery getHighlightQuery(Highlight highlight, @Nullable Class type) { + + Assert.notNull(highlight, "highlight must not be null"); + + HighlightBuilder highlightBuilder = new HighlightBuilder(); + + addParameters(highlight.parameters(), highlightBuilder, type); + + for (HighlightField highlightField : highlight.fields()) { + String mappedName = mapFieldName(highlightField.name(), type); + HighlightBuilder.Field field = new HighlightBuilder.Field(mappedName); + + addParameters(highlightField.parameters(), field, type); + + highlightBuilder.field(field); + } + + return new HighlightQuery(highlightBuilder); + } + + private void addParameters(HighlightParameters parameters, AbstractHighlighterBuilder builder, Class type) { + + if (StringUtils.hasLength(parameters.boundaryChars())) { + builder.boundaryChars(parameters.boundaryChars().toCharArray()); + } + + if (parameters.boundaryMaxScan() > -1) { + builder.boundaryMaxScan(parameters.boundaryMaxScan()); + } + + if (StringUtils.hasLength(parameters.boundaryScanner())) { + builder.boundaryScannerType(parameters.boundaryScanner()); + } + + if (StringUtils.hasLength(parameters.boundaryScannerLocale())) { + builder.boundaryScannerLocale(parameters.boundaryScannerLocale()); + } + + if (parameters.forceSource()) { // default is false + builder.forceSource(parameters.forceSource()); + } + + if (StringUtils.hasLength(parameters.fragmenter())) { + builder.fragmenter(parameters.fragmenter()); + } + + if (parameters.fragmentSize() > -1) { + builder.fragmentSize(parameters.fragmentSize()); + } + + if (parameters.noMatchSize() > -1) { + builder.noMatchSize(parameters.noMatchSize()); + } + + if (parameters.numberOfFragments() > -1) { + builder.numOfFragments(parameters.numberOfFragments()); + } + + if (StringUtils.hasLength(parameters.order())) { + builder.order(parameters.order()); + } + + if (parameters.phraseLimit() > -1) { + builder.phraseLimit(parameters.phraseLimit()); + } + + if (parameters.preTags().length > 0) { + builder.preTags(parameters.preTags()); + } + + if (parameters.postTags().length > 0) { + builder.postTags(parameters.postTags()); + } + + if (!parameters.requireFieldMatch()) { // default is true + builder.requireFieldMatch(parameters.requireFieldMatch()); + } + + if (StringUtils.hasLength(parameters.type())) { + builder.highlighterType(parameters.type()); + } + + if (builder instanceof HighlightBuilder) { + HighlightBuilder highlightBuilder = (HighlightBuilder) builder; + + if (StringUtils.hasLength(parameters.encoder())) { + highlightBuilder.encoder(parameters.encoder()); + } + + if (StringUtils.hasLength(parameters.tagsSchema())) { + highlightBuilder.tagsSchema(parameters.tagsSchema()); + } + } + + if (builder instanceof HighlightBuilder.Field) { + HighlightBuilder.Field field = (HighlightBuilder.Field) builder; + + if (parameters.fragmentOffset() > -1) { + field.fragmentOffset(parameters.fragmentOffset()); + } + + if (parameters.matchedFields().length > 0) { + field.matchedFields(Arrays.stream(parameters.matchedFields()) // + .map(fieldName -> mapFieldName(fieldName, type)) // + .collect(Collectors.toList()) // + .toArray(new String[] {})); // + } + } + } + + private String mapFieldName(String fieldName, @Nullable Class type) { + + if (type != null) { + ElasticsearchPersistentEntity persistentEntity = mappingContext.getPersistentEntity(type); + + if (persistentEntity != null) { + ElasticsearchPersistentProperty persistentProperty = persistentEntity.getPersistentProperty(fieldName); + + if (persistentProperty != null) { + return persistentProperty.getFieldName(); + } + } + } + + return fieldName; + } +} diff --git a/src/main/java/org/springframework/data/elasticsearch/core/query/Query.java b/src/main/java/org/springframework/data/elasticsearch/core/query/Query.java index 581b6734e..0dde4e6a1 100644 --- a/src/main/java/org/springframework/data/elasticsearch/core/query/Query.java +++ b/src/main/java/org/springframework/data/elasticsearch/core/query/Query.java @@ -17,6 +17,7 @@ package org.springframework.data.elasticsearch.core.query; import java.util.Collection; import java.util.List; +import java.util.Optional; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.support.IndicesOptions; @@ -24,6 +25,7 @@ import org.elasticsearch.index.query.QueryBuilders; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; +import org.springframework.lang.Nullable; /** * Query @@ -185,4 +187,19 @@ public interface Query { return null; } + /** + * Sets the {@link HighlightQuery}.* + * + * @param highlightQuery the query to set + * @since 4.0 + */ + void setHighlightQuery(@Nullable HighlightQuery highlightQuery); + + /** + * @return the optional set {@link HighlightQuery}. + * @since 4.0 + */ + default Optional getHighlightQuery() { + return Optional.empty(); + } } diff --git a/src/main/java/org/springframework/data/elasticsearch/repository/query/AbstractReactiveElasticsearchRepositoryQuery.java b/src/main/java/org/springframework/data/elasticsearch/repository/query/AbstractReactiveElasticsearchRepositoryQuery.java index eb4808bd7..6565c7944 100644 --- a/src/main/java/org/springframework/data/elasticsearch/repository/query/AbstractReactiveElasticsearchRepositoryQuery.java +++ b/src/main/java/org/springframework/data/elasticsearch/repository/query/AbstractReactiveElasticsearchRepositoryQuery.java @@ -84,6 +84,10 @@ abstract class AbstractReactiveElasticsearchRepositoryQuery implements Repositor Query query = createQuery( new ConvertingParameterAccessor(elasticsearchOperations.getElasticsearchConverter(), parameterAccessor)); + if (queryMethod.hasAnnotatedHighlight()) { + query.setHighlightQuery(queryMethod.getAnnotatedHighlightQuery()); + } + Class targetType = processor.getReturnedType().getTypeToRead(); String indexName = queryMethod.getEntityInformation().getIndexName(); String indexTypeName = queryMethod.getEntityInformation().getIndexTypeName(); diff --git a/src/main/java/org/springframework/data/elasticsearch/repository/query/ElasticsearchPartQuery.java b/src/main/java/org/springframework/data/elasticsearch/repository/query/ElasticsearchPartQuery.java index b6be9d570..51c36f5c0 100644 --- a/src/main/java/org/springframework/data/elasticsearch/repository/query/ElasticsearchPartQuery.java +++ b/src/main/java/org/springframework/data/elasticsearch/repository/query/ElasticsearchPartQuery.java @@ -58,13 +58,19 @@ public class ElasticsearchPartQuery extends AbstractElasticsearchRepositoryQuery @Override public Object execute(Object[] parameters) { + Class clazz = queryMethod.getEntityInformation().getJavaType(); ParametersParameterAccessor accessor = new ParametersParameterAccessor(queryMethod.getParameters(), parameters); + CriteriaQuery query = createQuery(accessor); + Assert.notNull(query, "unsupported query"); - Class clazz = queryMethod.getEntityInformation().getJavaType(); elasticsearchConverter.updateQuery(query, clazz); + if (queryMethod.hasAnnotatedHighlight()) { + query.setHighlightQuery(queryMethod.getAnnotatedHighlightQuery()); + } + IndexCoordinates index = elasticsearchOperations.getIndexCoordinatesFor(clazz); Object result = null; diff --git a/src/main/java/org/springframework/data/elasticsearch/repository/query/ElasticsearchQueryMethod.java b/src/main/java/org/springframework/data/elasticsearch/repository/query/ElasticsearchQueryMethod.java index 2705e7b69..3b1d28876 100644 --- a/src/main/java/org/springframework/data/elasticsearch/repository/query/ElasticsearchQueryMethod.java +++ b/src/main/java/org/springframework/data/elasticsearch/repository/query/ElasticsearchQueryMethod.java @@ -21,15 +21,19 @@ import java.util.Collection; import java.util.stream.Stream; import org.springframework.core.annotation.AnnotationUtils; +import org.springframework.data.elasticsearch.annotations.Highlight; import org.springframework.data.elasticsearch.annotations.Query; import org.springframework.data.elasticsearch.core.SearchHit; import org.springframework.data.elasticsearch.core.SearchHits; import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentEntity; import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentProperty; +import org.springframework.data.elasticsearch.core.query.HighlightQuery; +import org.springframework.data.elasticsearch.core.query.HighlightQueryBuilder; import org.springframework.data.mapping.context.MappingContext; import org.springframework.data.projection.ProjectionFactory; import org.springframework.data.repository.core.RepositoryMetadata; import org.springframework.data.repository.query.QueryMethod; +import org.springframework.data.util.Lazy; import org.springframework.lang.Nullable; import org.springframework.util.Assert; import org.springframework.util.ClassUtils; @@ -46,10 +50,12 @@ import org.springframework.util.ClassUtils; */ public class ElasticsearchQueryMethod extends QueryMethod { - private final Method method; // private in base class, but needed here as well - private final Query queryAnnotation; private final MappingContext, ElasticsearchPersistentProperty> mappingContext; private @Nullable ElasticsearchEntityMetadata metadata; + private final Method method; // private in base class, but needed here as well + private final Query queryAnnotation; + private final Highlight highlightAnnotation; + private final Lazy highlightQueryLazy = Lazy.of(this::createAnnotatedHighlightQuery); public ElasticsearchQueryMethod(Method method, RepositoryMetadata repositoryMetadata, ProjectionFactory factory, MappingContext, ElasticsearchPersistentProperty> mappingContext) { @@ -61,6 +67,7 @@ public class ElasticsearchQueryMethod extends QueryMethod { this.method = method; this.mappingContext = mappingContext; this.queryAnnotation = method.getAnnotation(Query.class); + this.highlightAnnotation = method.getAnnotation(Highlight.class); } public boolean hasAnnotatedQuery() { @@ -71,6 +78,30 @@ public class ElasticsearchQueryMethod extends QueryMethod { return (String) AnnotationUtils.getValue(queryAnnotation, "value"); } + /** + * @return true if there is a {@link Highlight} annotation present. + * @since 4.0 + */ + public boolean hasAnnotatedHighlight() { + return highlightAnnotation != null; + } + + /** + * @return a {@link HighlightQuery} built from the {@link Highlight} annotation. + * @throws IllegalArgumentException if no {@link Highlight} annotation is present on the method + * @see #hasAnnotatedHighlight() + */ + public HighlightQuery getAnnotatedHighlightQuery() { + + Assert.isTrue(hasAnnotatedHighlight(), "no Highlight annotation present on " + getName()); + + return highlightQueryLazy.get(); + } + + private HighlightQuery createAnnotatedHighlightQuery() { + return new HighlightQueryBuilder(mappingContext).getHighlightQuery(highlightAnnotation, getDomainClass()); + } + /** * @return the {@link ElasticsearchEntityMetadata} for the query methods {@link #getReturnedObjectType() return type}. * @since 3.2 diff --git a/src/main/java/org/springframework/data/elasticsearch/repository/query/ElasticsearchStringQuery.java b/src/main/java/org/springframework/data/elasticsearch/repository/query/ElasticsearchStringQuery.java index 567a4696f..82dfe9425 100644 --- a/src/main/java/org/springframework/data/elasticsearch/repository/query/ElasticsearchStringQuery.java +++ b/src/main/java/org/springframework/data/elasticsearch/repository/query/ElasticsearchStringQuery.java @@ -69,9 +69,17 @@ public class ElasticsearchStringQuery extends AbstractElasticsearchRepositoryQue @Override public Object execute(Object[] parameters) { - ParametersParameterAccessor accessor = new ParametersParameterAccessor(queryMethod.getParameters(), parameters); - StringQuery stringQuery = createQuery(accessor); Class clazz = queryMethod.getEntityInformation().getJavaType(); + ParametersParameterAccessor accessor = new ParametersParameterAccessor(queryMethod.getParameters(), parameters); + + StringQuery stringQuery = createQuery(accessor); + + Assert.notNull(stringQuery, "unsupported query"); + + if (queryMethod.hasAnnotatedHighlight()) { + stringQuery.setHighlightQuery(queryMethod.getAnnotatedHighlightQuery()); + } + IndexCoordinates index = elasticsearchOperations.getIndexCoordinatesFor(clazz); Object result = null; diff --git a/src/main/java/org/springframework/data/elasticsearch/repository/query/ReactivePartTreeElasticsearchQuery.java b/src/main/java/org/springframework/data/elasticsearch/repository/query/ReactivePartTreeElasticsearchQuery.java index ee90dfef0..e723e275c 100644 --- a/src/main/java/org/springframework/data/elasticsearch/repository/query/ReactivePartTreeElasticsearchQuery.java +++ b/src/main/java/org/springframework/data/elasticsearch/repository/query/ReactivePartTreeElasticsearchQuery.java @@ -46,6 +46,7 @@ public class ReactivePartTreeElasticsearchQuery extends AbstractReactiveElastics if (tree.isLimiting()) { query.setMaxResults(tree.getMaxResults()); } + return query; } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentEntityTests.java b/src/test/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentEntityTests.java index 735d9428d..1180009fd 100644 --- a/src/test/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentEntityTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentEntityTests.java @@ -17,10 +17,10 @@ package org.springframework.data.elasticsearch.core.mapping; import static org.assertj.core.api.Assertions.*; -import java.beans.IntrospectionException; - import org.junit.jupiter.api.Test; +import org.springframework.data.annotation.Id; import org.springframework.data.annotation.Version; +import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.Score; import org.springframework.data.mapping.MappingException; import org.springframework.data.mapping.model.Property; @@ -79,6 +79,21 @@ public class SimpleElasticsearchPersistentEntityTests { .withMessageContaining("second"); } + @Test + void shouldFindPropertiesByMappedName() { + + SimpleElasticsearchMappingContext context = new SimpleElasticsearchMappingContext(); + SimpleElasticsearchPersistentEntity persistentEntity = context + .getRequiredPersistentEntity(FieldNameEntity.class); + + ElasticsearchPersistentProperty persistentProperty = persistentEntity + .getPersistentPropertyWithFieldName("renamed-field"); + + assertThat(persistentProperty).isNotNull(); + assertThat(persistentProperty.getName()).isEqualTo("renamedField"); + assertThat(persistentProperty.getFieldName()).isEqualTo("renamed-field"); + } + private static SimpleElasticsearchPersistentProperty createProperty(SimpleElasticsearchPersistentEntity entity, String field) { @@ -130,4 +145,9 @@ public class SimpleElasticsearchPersistentEntityTests { @Score float first; @Score float second; } + + private static class FieldNameEntity { + @Id private String id; + @Field(name = "renamed-field") private String renamedField; + } } diff --git a/src/test/java/org/springframework/data/elasticsearch/core/query/HighlightQueryBuilderTests.java b/src/test/java/org/springframework/data/elasticsearch/core/query/HighlightQueryBuilderTests.java new file mode 100644 index 000000000..7b1d09f17 --- /dev/null +++ b/src/test/java/org/springframework/data/elasticsearch/core/query/HighlightQueryBuilderTests.java @@ -0,0 +1,141 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch.core.query; + +import static org.skyscreamer.jsonassert.JSONAssert.*; + +import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.json.JSONException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.data.annotation.Id; +import org.springframework.data.elasticsearch.annotations.Document; +import org.springframework.data.elasticsearch.annotations.Field; +import org.springframework.data.elasticsearch.annotations.Highlight; +import org.springframework.data.elasticsearch.annotations.HighlightField; +import org.springframework.data.elasticsearch.annotations.HighlightParameters; +import org.springframework.data.elasticsearch.core.ResourceUtil; +import org.springframework.data.elasticsearch.core.mapping.SimpleElasticsearchMappingContext; +import org.springframework.util.Assert; + +/** + * @author Peter-Josef Meisch + */ +@ExtendWith(MockitoExtension.class) +class HighlightQueryBuilderTests { + + private final SimpleElasticsearchMappingContext context = new SimpleElasticsearchMappingContext(); + + private HighlightQueryBuilder highlightQueryBuilder = new HighlightQueryBuilder(context); + + @Test + void shouldProcessAnnotationWithNoParameters() throws NoSuchMethodException, JSONException { + Highlight highlight = getAnnotation("annotatedMethod"); + String expected = ResourceUtil.readFileFromClasspath("/highlights/highlights.json"); + + HighlightBuilder highlightBuilder = highlightQueryBuilder.getHighlightQuery(highlight, HighlightEntity.class) + .getHighlightBuilder(); + String actualStr = highlightBuilder.toString(); + assertEquals(expected, actualStr, false); + + } + + @Test + void shouldProcessAnnotationWithParameters() throws NoSuchMethodException, JSONException { + Highlight highlight = getAnnotation("annotatedMethodWithManyValue"); + String expected = ResourceUtil.readFileFromClasspath("/highlights/highlights-with-parameters.json"); + + HighlightBuilder highlightBuilder = highlightQueryBuilder.getHighlightQuery(highlight, HighlightEntity.class) + .getHighlightBuilder(); + String actualStr = highlightBuilder.toString(); + + assertEquals(expected, actualStr, true); + } + + private Highlight getAnnotation(String methodName) throws NoSuchMethodException { + Highlight highlight = HighlightQueryBuilderTests.class.getDeclaredMethod(methodName).getAnnotation(Highlight.class); + + Assert.notNull(highlight, "no highlight annotation found"); + + return highlight; + } + + /** + * The annotation values on this method are just random values. The field has just one common parameters and the field + * specific, the whole bunch pf parameters is tested on the top level. tagsSchema cannot be tested together with + * preTags and postTags, ist it sets it's own values for these. + */ + // region test data + @Highlight(fields = { @HighlightField(name = "someField") }) + private void annotatedMethod() {} + + @Highlight( // + parameters = @HighlightParameters( // + boundaryChars = "#+*", // + boundaryMaxScan = 7, // + boundaryScanner = "chars", // + boundaryScannerLocale = "de-DE", // + encoder = "html", // + forceSource = true, // + fragmenter = "span", // + noMatchSize = 2, // + numberOfFragments = 3, // + fragmentSize = 5, // + order = "score", // + phraseLimit = 42, // + preTags = { "", "" }, // + postTags = { "", "" }, // + requireFieldMatch = false, // + type = "plain" // + ), // + fields = { // + @HighlightField( // + name = "someField", // + parameters = @HighlightParameters( // + fragmentOffset = 3, // + matchedFields = { "someField", "otherField" }, // + numberOfFragments = 4) // + // + ) // + } // + ) // + private void annotatedMethodWithManyValue() {} + + @Document(indexName = "dont-care") + private static class HighlightEntity { + @Id private String id; + @Field(name = "some-field") private String someField; + @Field(name = "other-field") private String otherField; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getSomeField() { + return someField; + } + + public void setSomeField(String someField) { + this.someField = someField; + } + } + // endregion +} diff --git a/src/test/java/org/springframework/data/elasticsearch/repositories/custommethod/CustomMethodRepositoryBaseTests.java b/src/test/java/org/springframework/data/elasticsearch/repositories/custommethod/CustomMethodRepositoryBaseTests.java index 943127887..50ea5f11a 100644 --- a/src/test/java/org/springframework/data/elasticsearch/repositories/custommethod/CustomMethodRepositoryBaseTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/repositories/custommethod/CustomMethodRepositoryBaseTests.java @@ -46,6 +46,8 @@ import org.springframework.data.domain.Sort; import org.springframework.data.domain.Sort.Order; import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Field; +import org.springframework.data.elasticsearch.annotations.Highlight; +import org.springframework.data.elasticsearch.annotations.HighlightField; import org.springframework.data.elasticsearch.annotations.Query; import org.springframework.data.elasticsearch.core.ElasticsearchOperations; import org.springframework.data.elasticsearch.core.IndexOperations; @@ -1416,6 +1418,32 @@ public abstract class CustomMethodRepositoryBaseTests { assertThat(searchHits.getTotalHits()).isEqualTo(20); } + @Test // DATAES-372 + void shouldReturnHighlightsOnAnnotatedMethod() { + List entities = createSampleEntities("abc", 2); + repository.saveAll(entities); + + // when + SearchHits searchHits = repository.queryByType("abc"); + + assertThat(searchHits.getTotalHits()).isEqualTo(2); + SearchHit searchHit = searchHits.getSearchHit(0); + assertThat(searchHit.getHighlightField("type")).hasSize(1).contains("abc"); + } + + @Test // DATAES-372 + void shouldReturnHighlightsOnAnnotatedStringQueryMethod() { + List entities = createSampleEntities("abc", 2); + repository.saveAll(entities); + + // when + SearchHits searchHits = repository.queryByString("abc"); + + assertThat(searchHits.getTotalHits()).isEqualTo(2); + SearchHit searchHit = searchHits.getSearchHit(0); + assertThat(searchHit.getHighlightField("type")).hasSize(1).contains("abc"); + } + private List createSampleEntities(String type, int numberOfEntities) { List entities = new ArrayList<>(); @@ -1552,14 +1580,17 @@ public abstract class CustomMethodRepositoryBaseTests { long countByLocationNear(GeoPoint point, String distance); + @Highlight(fields = { @HighlightField(name = "type") }) SearchHits queryByType(String type); @Query("{\"bool\": {\"must\": [{\"term\": {\"type\": \"?0\"}}]}}") + @Highlight(fields = { @HighlightField(name = "type") }) SearchHits queryByString(String type); - List> queryByMessage(String type); + List> queryByMessage(String message); + + Stream> readByMessage(String message); - Stream> readByMessage(String type); } /** diff --git a/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryTests.java b/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryTests.java index c1c933eed..9534fc00d 100644 --- a/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryTests.java +++ b/src/test/java/org/springframework/data/elasticsearch/repository/support/SimpleReactiveElasticsearchRepositoryTests.java @@ -33,6 +33,7 @@ import java.lang.Long; import java.lang.Object; import java.util.Arrays; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.UUID; import java.util.stream.IntStream; @@ -58,6 +59,8 @@ import org.springframework.data.domain.Sort.Order; import org.springframework.data.elasticsearch.TestUtils; import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Field; +import org.springframework.data.elasticsearch.annotations.Highlight; +import org.springframework.data.elasticsearch.annotations.HighlightField; import org.springframework.data.elasticsearch.annotations.Query; import org.springframework.data.elasticsearch.annotations.Score; import org.springframework.data.elasticsearch.client.reactive.ReactiveElasticsearchClient; @@ -206,7 +209,7 @@ public class SimpleReactiveElasticsearchRepositoryTests { SampleEntity.builder().id("id-two").message("message").build(), // SampleEntity.builder().id("id-three").message("message").build()); - repository.queryByMessageWithString("message") // + repository.queryAllByMessage("message") // .as(StepVerifier::create) // .expectNextMatches(searchHit -> SearchHit.class.isAssignableFrom(searchHit.getClass()))// .expectNextCount(2) // @@ -220,13 +223,47 @@ public class SimpleReactiveElasticsearchRepositoryTests { SampleEntity.builder().id("id-two").message("message").build(), // SampleEntity.builder().id("id-three").message("message").build()); - repository.queryAllByMessage("message") // + repository.queryByMessageWithString("message") // .as(StepVerifier::create) // .expectNextMatches(searchHit -> SearchHit.class.isAssignableFrom(searchHit.getClass()))// .expectNextCount(2) // .verifyComplete(); } + @Test // DATAES-372 + void shouldReturnHighlightsOnAnnotatedMethod() throws IOException { + + bulkIndex(SampleEntity.builder().id("id-one").message("message").build(), // + SampleEntity.builder().id("id-two").message("message").build(), // + SampleEntity.builder().id("id-three").message("message").build()); + + repository.queryAllByMessage("message") // + .as(StepVerifier::create) // + .expectNextMatches(searchHit -> { + List hitHighlightField = searchHit.getHighlightField("message"); + return hitHighlightField.size() == 1 && hitHighlightField.get(0).equals("message"); + }) // + .expectNextCount(2) // + .verifyComplete(); + } + + @Test // DATAES-372 + void shouldReturnHighlightsOnAnnotatedStringQueryMethod() throws IOException { + + bulkIndex(SampleEntity.builder().id("id-one").message("message").build(), // + SampleEntity.builder().id("id-two").message("message").build(), // + SampleEntity.builder().id("id-three").message("message").build()); + + repository.queryByMessageWithString("message") // + .as(StepVerifier::create) // + .expectNextMatches(searchHit -> { + List hitHighlightField = searchHit.getHighlightField("message"); + return hitHighlightField.size() == 1 && hitHighlightField.get(0).equals("message"); + }) // + .expectNextCount(2) // + .verifyComplete(); + } + @Test // DATAES-519 public void countShouldReturnZeroWhenIndexDoesNotExist() { repository.count().as(StepVerifier::create).expectNext(0L).verifyComplete(); @@ -535,9 +572,11 @@ public class SimpleReactiveElasticsearchRepositoryTests { Flux findAllByMessage(Publisher message); + @Highlight(fields = { @HighlightField(name = "message") }) Flux> queryAllByMessage(String message); @Query("{\"bool\": {\"must\": [{\"term\": {\"message\": \"?0\"}}]}}") + @Highlight(fields = { @HighlightField(name = "message") }) Flux> queryByMessageWithString(String message); @Query("{ \"bool\" : { \"must\" : { \"term\" : { \"message\" : \"?0\" } } } }") diff --git a/src/test/resources/highlights/highlights-with-parameters.json b/src/test/resources/highlights/highlights-with-parameters.json new file mode 100644 index 000000000..566d2996f --- /dev/null +++ b/src/test/resources/highlights/highlights-with-parameters.json @@ -0,0 +1,34 @@ +{ + "boundary_chars": "#+*", + "boundary_max_scan": 7, + "boundary_scanner": "CHARS", + "boundary_scanner_locale": "de-DE", + "encoder": "html", + "force_source": true, + "fragmenter": "span", + "fragment_size": 5, + "no_match_size": 2, + "number_of_fragments": 3, + "order": "score", + "phrase_limit": 42, + "pre_tags": [ + "", + "" + ], + "post_tags": [ + "", + "" + ], + "require_field_match": false, + "type": "plain", + "fields": { + "some-field": { + "fragment_offset": 3, + "number_of_fragments": 4, + "matched_fields": [ + "some-field", + "other-field" + ] + } + } +} diff --git a/src/test/resources/highlights/highlights.json b/src/test/resources/highlights/highlights.json new file mode 100644 index 000000000..0a2c7eeca --- /dev/null +++ b/src/test/resources/highlights/highlights.json @@ -0,0 +1,5 @@ +{ + "fields": { + "some-field": {} + } +}