Cwikius-Spring/spring-data-elasticsearch
1
0
Fork 0
mirror of https://github.com/spring-projects/spring-data-elasticsearch.git synced 2025-06-08 13:12:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support field exclusion from source.

Browse Source 
Original Pull Request #1962 
Closes #769
This commit is contained in:
Peter-Josef Meisch 2021-10-16 13:01:14 +02:00 committed by GitHub
parent 59fdbbeb19
commit 288705ca72
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 482 additions and 360 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/java/org/springframework/data/elasticsearch/annotations/Field.java
View File

@ -203,4 +203,12 @@ public @interface Field {
* @since 4.3
*/
Dynamic dynamic() default Dynamic.INHERIT;
/**
* marks this field to be excluded from the _source in Elasticsearch
* (https://www.elastic.co/guide/en/elasticsearch/reference/7.15.0/mapping-source-field.html#include-exclude)
*
* @since 4.3
*/
boolean excludeFromSource() default false;
}

94
src/main/java/org/springframework/data/elasticsearch/core/index/MappingBuilder.java
View File

@ -21,8 +21,10 @@ import static org.springframework.util.StringUtils.*;
import java.io.IOException;
import java.lang.annotation.Annotation;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;
import org.slf4j.Logger;
@ -102,12 +104,12 @@ public class MappingBuilder {
private static final String NUMERIC_DETECTION = "numeric_detection";
private static final String DYNAMIC_DATE_FORMATS = "dynamic_date_formats";
private static final String RUNTIME = "runtime";
private static final String SOURCE = "_source";
private static final String SOURCE_EXCLUDES = "excludes";
protected final ElasticsearchConverter elasticsearchConverter;
private final ObjectMapper objectMapper = new ObjectMapper();
private boolean writeTypeHints = true;
public MappingBuilder(ElasticsearchConverter elasticsearchConverter) {
this.elasticsearchConverter = elasticsearchConverter;
}
@ -126,6 +128,37 @@ public class MappingBuilder {
return buildPropertyMapping(entity, getRuntimeFields(entity));
}
protected String buildPropertyMapping(ElasticsearchPersistentEntity<?> entity,
@Nullable org.springframework.data.elasticsearch.core.document.Document runtimeFields) {
InternalBuilder internalBuilder = new InternalBuilder();
return internalBuilder.buildPropertyMapping(entity, runtimeFields);
}
@Nullable
private org.springframework.data.elasticsearch.core.document.Document getRuntimeFields(
@Nullable ElasticsearchPersistentEntity<?> entity) {
if (entity != null) {
Mapping mappingAnnotation = entity.findAnnotation(Mapping.class);
if (mappingAnnotation != null) {
String runtimeFieldsPath = mappingAnnotation.runtimeFieldsPath();
if (hasText(runtimeFieldsPath)) {
String jsonString = ResourceUtil.readFileFromClasspath(runtimeFieldsPath);
return org.springframework.data.elasticsearch.core.document.Document.parse(jsonString);
}
}
}
return null;
}
private class InternalBuilder {
private boolean writeTypeHints = true;
private List<String> excludeFromSource = new ArrayList<>();
private String nestedPropertyPrefix = "";
protected String buildPropertyMapping(ElasticsearchPersistentEntity<?> entity,
@Nullable org.springframework.data.elasticsearch.core.document.Document runtimeFields) {
@ -138,8 +171,14 @@ public class MappingBuilder {
// Dynamic templates
addDynamicTemplatesMapping(objectNode, entity);
mapEntity(objectNode, entity, true, "", false, FieldType.Auto, null, entity.findAnnotation(DynamicMapping.class),
runtimeFields);
mapEntity(objectNode, entity, true, "", false, FieldType.Auto, null,
entity.findAnnotation(DynamicMapping.class), runtimeFields);
if (!excludeFromSource.isEmpty()) {
ObjectNode sourceNode = objectNode.putObject(SOURCE);
ArrayNode excludes = sourceNode.putArray(SOURCE_EXCLUDES);
excludeFromSource.stream().map(TextNode::new).forEach(excludes::add);
}
return objectMapper.writer().writeValueAsString(objectNode);
} catch (IOException e) {
@ -157,10 +196,10 @@ public class MappingBuilder {
}
}
private void mapEntity(ObjectNode objectNode, @Nullable ElasticsearchPersistentEntity<?> entity, boolean isRootObject,
String nestedObjectFieldName, boolean nestedOrObjectField, FieldType fieldType,
@Nullable Field parentFieldAnnotation, @Nullable DynamicMapping dynamicMapping, @Nullable Document runtimeFields)
throws IOException {
private void mapEntity(ObjectNode objectNode, @Nullable ElasticsearchPersistentEntity<?> entity,
boolean isRootObject, String nestedObjectFieldName, boolean nestedOrObjectField, FieldType fieldType,
@Nullable Field parentFieldAnnotation, @Nullable DynamicMapping dynamicMapping,
@Nullable Document runtimeFields) throws IOException {
if (entity != null && entity.isAnnotationPresent(Mapping.class)) {
Mapping mappingAnnotation = entity.getRequiredAnnotation(Mapping.class);
@ -179,8 +218,8 @@ public class MappingBuilder {
}
if (mappingAnnotation.dynamicDateFormats().length > 0) {
objectNode.putArray(DYNAMIC_DATE_FORMATS).addAll(
Arrays.stream(mappingAnnotation.dynamicDateFormats()).map(TextNode::valueOf).collect(Collectors.toList()));
objectNode.putArray(DYNAMIC_DATE_FORMATS).addAll(Arrays.stream(mappingAnnotation.dynamicDateFormats())
.map(TextNode::valueOf).collect(Collectors.toList()));
}
if (runtimeFields != null) {
@ -241,24 +280,6 @@ public class MappingBuilder {
}
}
@Nullable
private org.springframework.data.elasticsearch.core.document.Document getRuntimeFields(
@Nullable ElasticsearchPersistentEntity<?> entity) {
if (entity != null) {
Mapping mappingAnnotation = entity.findAnnotation(Mapping.class);
if (mappingAnnotation != null) {
String runtimeFieldsPath = mappingAnnotation.runtimeFieldsPath();
if (hasText(runtimeFieldsPath)) {
String jsonString = ResourceUtil.readFileFromClasspath(runtimeFieldsPath);
return org.springframework.data.elasticsearch.core.document.Document.parse(jsonString);
}
}
}
return null;
}
private void buildPropertyMapping(ObjectNode propertiesNode, boolean isRootObject,
ElasticsearchPersistentProperty property) throws IOException {
@ -297,7 +318,15 @@ public class MappingBuilder {
addJoinFieldMapping(propertiesNode, property);
}
String nestedPropertyPath = nestedPropertyPrefix.isEmpty() ? property.getFieldName()
: nestedPropertyPrefix + '.' + property.getFieldName();
Field fieldAnnotation = property.findAnnotation(Field.class);
if (fieldAnnotation != null && fieldAnnotation.excludeFromSource()) {
excludeFromSource.add(nestedPropertyPath);
}
boolean isCompletionProperty = property.isCompletionProperty();
boolean isNestedOrObjectProperty = isNestedOrObjectProperty(property);
DynamicMapping dynamicMapping = property.findAnnotation(DynamicMapping.class);
@ -314,8 +343,13 @@ public class MappingBuilder {
? elasticsearchConverter.getMappingContext().getPersistentEntity(iterator.next())
: null;
String currentNestedPropertyPrefix = nestedPropertyPrefix;
nestedPropertyPrefix = nestedPropertyPath;
mapEntity(propertiesNode, persistentEntity, false, property.getFieldName(), true, fieldAnnotation.type(),
fieldAnnotation, dynamicMapping, null);
nestedPropertyPrefix = currentNestedPropertyPrefix;
return;
}
}
@ -487,7 +521,8 @@ public class MappingBuilder {
* @throws IOException
*/
private void addMultiFieldMapping(ObjectNode propertyNode, ElasticsearchPersistentProperty property,
MultiField annotation, boolean nestedOrObjectField, @Nullable DynamicMapping dynamicMapping) throws IOException {
MultiField annotation, boolean nestedOrObjectField, @Nullable DynamicMapping dynamicMapping)
throws IOException {
// main field
ObjectNode mainFieldNode = objectMapper.createObjectNode();
@ -571,3 +606,4 @@ public class MappingBuilder {
&& (FieldType.Nested == fieldAnnotation.type() || FieldType.Object == fieldAnnotation.type());
}
}
}

25
src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java
View File

@ -26,6 +26,7 @@ import java.lang.Integer;
import java.lang.Object;
import java.math.BigDecimal;
import java.time.Instant;
import java.time.LocalDate;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
@ -47,13 +48,13 @@ import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
import org.springframework.data.elasticsearch.core.IndexOperations;
import org.springframework.data.elasticsearch.core.MappingContextBaseTests;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.data.elasticsearch.core.suggest.Completion;
import org.springframework.data.elasticsearch.core.geo.GeoPoint;
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
import org.springframework.data.elasticsearch.core.query.IndexQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.data.elasticsearch.core.query.SeqNoPrimaryTerm;
import org.springframework.data.elasticsearch.core.suggest.Completion;
import org.springframework.data.elasticsearch.junit.jupiter.ElasticsearchRestTemplateConfiguration;
import org.springframework.data.elasticsearch.junit.jupiter.SpringIntegrationTest;
import org.springframework.data.geo.Box;
@ -325,6 +326,16 @@ public class MappingBuilderIntegrationTests extends MappingContextBaseTests {
}
@Test // #796
@DisplayName("should write source excludes")
void shouldWriteSourceExcludes() {
IndexOperations indexOps = operations.indexOps(ExcludedFieldEntity.class);
indexOps.create();
indexOps.putMapping();
}
// region entities
@Document(indexName = "ignore-above-index")
static class IgnoreAboveEntity {
@ -1172,6 +1183,18 @@ public class MappingBuilderIntegrationTests extends MappingContextBaseTests {
@Field(type = Date, format = DateFormat.epoch_millis, name = "@timestamp") @Nullable private Instant timestamp;
}
@Document(indexName = "fields-excluded-from-source")
private static class ExcludedFieldEntity {
@Id @Nullable private String id;
@Nullable @Field(name = "excluded-date", type = Date, format = DateFormat.date,
excludeFromSource = true) private LocalDate excludedDate;
@Nullable @Field(type = Nested) private NestedExcludedFieldEntity nestedEntity;
}
private static class NestedExcludedFieldEntity {
@Nullable @Field(name = "excluded-text", type = Text, excludeFromSource = true) private String excludedText;
}
// endregion
}

57
src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderUnitTests.java
View File

@ -42,10 +42,10 @@ import org.springframework.data.annotation.Id;
import org.springframework.data.annotation.Transient;
import org.springframework.data.elasticsearch.annotations.*;
import org.springframework.data.elasticsearch.core.MappingContextBaseTests;
import org.springframework.data.elasticsearch.core.suggest.Completion;
import org.springframework.data.elasticsearch.core.geo.GeoPoint;
import org.springframework.data.elasticsearch.core.mapping.SimpleElasticsearchMappingContext;
import org.springframework.data.elasticsearch.core.query.SeqNoPrimaryTerm;
import org.springframework.data.elasticsearch.core.suggest.Completion;
import org.springframework.data.geo.Box;
import org.springframework.data.geo.Circle;
import org.springframework.data.geo.Point;
@ -945,6 +945,49 @@ public class MappingBuilderUnitTests extends MappingContextBaseTests {
assertEquals(expected, mapping, true);
}
@Test // #796
@DisplayName("should add fields that are excluded from source")
void shouldAddFieldsThatAreExcludedFromSource() throws JSONException {
String expected = "{\n" + //
" \"properties\": {\n" + //
" \"_class\": {\n" + //
" \"type\": \"keyword\",\n" + //
" \"index\": false,\n" + //
" \"doc_values\": false\n" + //
" },\n" + //
" \"excluded-date\": {\n" + //
" \"type\": \"date\",\n" + //
" \"format\": \"date\"\n" + //
" },\n" + //
" \"nestedEntity\": {\n" + //
" \"type\": \"nested\",\n" + //
" \"properties\": {\n" + //
" \"_class\": {\n" + //
" \"type\": \"keyword\",\n" + //
" \"index\": false,\n" + //
" \"doc_values\": false\n" + //
" },\n" + //
" \"excluded-text\": {\n" + //
" \"type\": \"text\"\n" + //
" }\n" + //
" }\n" + //
" }\n" + //
" },\n" + //
" \"_source\": {\n" + //
" \"excludes\": [\n" + //
" \"excluded-date\",\n" + //
" \"nestedEntity.excluded-text\"\n" + //
" ]\n" + //
" }\n" + //
"}\n"; //
String mapping = getMappingBuilder().buildPropertyMapping(ExcludedFieldEntity.class);
assertEquals(expected, mapping, true);
}
// region entities
@Document(indexName = "ignore-above-index")
@ -1918,5 +1961,17 @@ public class MappingBuilderUnitTests extends MappingContextBaseTests {
@Id @Nullable private String id;
@Field(type = Date, format = DateFormat.epoch_millis, name = "@timestamp") @Nullable private Instant timestamp;
}
@Document(indexName = "fields-excluded-from-source")
private static class ExcludedFieldEntity {
@Id @Nullable private String id;
@Nullable @Field(name = "excluded-date", type = Date, format = DateFormat.date,
excludeFromSource = true) private LocalDate excludedDate;
@Nullable @Field(type = Nested) private NestedExcludedFieldEntity nestedEntity;
}
private static class NestedExcludedFieldEntity {
@Nullable @Field(name = "excluded-text", type = Text, excludeFromSource = true) private String excludedText;
}
// endregion
}