Cwikius-Spring/spring-data-elasticsearch
1
0
Fork 0
mirror of https://github.com/spring-projects/spring-data-elasticsearch.git synced 2025-06-08 13:12:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support field exclusion from source.

Browse Source 
Original Pull Request #1962 
Closes #769
This commit is contained in:
Peter-Josef Meisch 2021-10-16 13:01:14 +02:00 committed by GitHub
parent 59fdbbeb19
commit 288705ca72
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 482 additions and 360 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/java/org/springframework/data/elasticsearch/annotations/Field.java
View File

@ -203,4 +203,12 @@ public @interface Field {
* @since 4.3
*/
Dynamic dynamic() default Dynamic.INHERIT;
/**
* marks this field to be excluded from the _source in Elasticsearch
* (https://www.elastic.co/guide/en/elasticsearch/reference/7.15.0/mapping-source-field.html#include-exclude)
*
* @since 4.3
*/
boolean excludeFromSource() default false;
}

750
src/main/java/org/springframework/data/elasticsearch/core/index/MappingBuilder.java
View File

@ -21,8 +21,10 @@ import static org.springframework.util.StringUtils.*;
import java.io.IOException;
import java.lang.annotation.Annotation;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;
import org.slf4j.Logger;
@ -102,12 +104,12 @@ public class MappingBuilder {
private static final String NUMERIC_DETECTION = "numeric_detection";
private static final String DYNAMIC_DATE_FORMATS = "dynamic_date_formats";
private static final String RUNTIME = "runtime";
private static final String SOURCE = "_source";
private static final String SOURCE_EXCLUDES = "excludes";
protected final ElasticsearchConverter elasticsearchConverter;
private final ObjectMapper objectMapper = new ObjectMapper();
private boolean writeTypeHints = true;
public MappingBuilder(ElasticsearchConverter elasticsearchConverter) {
this.elasticsearchConverter = elasticsearchConverter;
}
@ -129,116 +131,8 @@ public class MappingBuilder {
protected String buildPropertyMapping(ElasticsearchPersistentEntity<?> entity,
@Nullable org.springframework.data.elasticsearch.core.document.Document runtimeFields) {
try {
writeTypeHints = entity.writeTypeHints();
ObjectNode objectNode = objectMapper.createObjectNode();
// Dynamic templates
addDynamicTemplatesMapping(objectNode, entity);
mapEntity(objectNode, entity, true, "", false, FieldType.Auto, null, entity.findAnnotation(DynamicMapping.class),
runtimeFields);
return objectMapper.writer().writeValueAsString(objectNode);
} catch (IOException e) {
throw new MappingException("could not build mapping", e);
}
}
private void writeTypeHintMapping(ObjectNode propertiesNode) throws IOException {
if (writeTypeHints) {
propertiesNode.set(TYPEHINT_PROPERTY, objectMapper.createObjectNode() //
.put(FIELD_PARAM_TYPE, TYPE_VALUE_KEYWORD) //
.put(FIELD_PARAM_INDEX, false) //
.put(FIELD_PARAM_DOC_VALUES, false));
}
}
private void mapEntity(ObjectNode objectNode, @Nullable ElasticsearchPersistentEntity<?> entity, boolean isRootObject,
String nestedObjectFieldName, boolean nestedOrObjectField, FieldType fieldType,
@Nullable Field parentFieldAnnotation, @Nullable DynamicMapping dynamicMapping, @Nullable Document runtimeFields)
throws IOException {
if (entity != null && entity.isAnnotationPresent(Mapping.class)) {
Mapping mappingAnnotation = entity.getRequiredAnnotation(Mapping.class);
if (!mappingAnnotation.enabled()) {
objectNode.put(MAPPING_ENABLED, false);
return;
}
if (mappingAnnotation.dateDetection() != Mapping.Detection.DEFAULT) {
objectNode.put(DATE_DETECTION, Boolean.parseBoolean(mappingAnnotation.dateDetection().name()));
}
if (mappingAnnotation.numericDetection() != Mapping.Detection.DEFAULT) {
objectNode.put(NUMERIC_DETECTION, Boolean.parseBoolean(mappingAnnotation.numericDetection().name()));
}
if (mappingAnnotation.dynamicDateFormats().length > 0) {
objectNode.putArray(DYNAMIC_DATE_FORMATS).addAll(
Arrays.stream(mappingAnnotation.dynamicDateFormats()).map(TextNode::valueOf).collect(Collectors.toList()));
}
if (runtimeFields != null) {
objectNode.set(RUNTIME, objectMapper.convertValue(runtimeFields, JsonNode.class));
}
}
boolean writeNestedProperties = !isRootObject && (isAnyPropertyAnnotatedWithField(entity) || nestedOrObjectField);
if (writeNestedProperties) {
String type = nestedOrObjectField ? fieldType.toString().toLowerCase()
: FieldType.Object.toString().toLowerCase();
ObjectNode nestedObjectNode = objectMapper.createObjectNode();
nestedObjectNode.put(FIELD_PARAM_TYPE, type);
if (nestedOrObjectField && FieldType.Nested == fieldType && parentFieldAnnotation != null
&& parentFieldAnnotation.includeInParent()) {
nestedObjectNode.put(FIELD_INCLUDE_IN_PARENT, true);
}
objectNode.set(nestedObjectFieldName, nestedObjectNode);
// now go on with the nested one
objectNode = nestedObjectNode;
}
if (entity != null && entity.dynamic() != Dynamic.INHERIT) {
objectNode.put(TYPE_DYNAMIC, entity.dynamic().name().toLowerCase());
} else if (dynamicMapping != null) {
objectNode.put(TYPE_DYNAMIC, dynamicMapping.value().name().toLowerCase());
}
ObjectNode propertiesNode = objectNode.putObject(FIELD_PROPERTIES);
writeTypeHintMapping(propertiesNode);
if (entity != null) {
entity.doWithProperties((PropertyHandler<ElasticsearchPersistentProperty>) property -> {
try {
if (property.isAnnotationPresent(Transient.class) || isInIgnoreFields(property, parentFieldAnnotation)) {
return;
}
if (property.isSeqNoPrimaryTermProperty()) {
if (property.isAnnotationPresent(Field.class)) {
logger.warn("Property {} of {} is annotated for inclusion in mapping, but its type is " + //
"SeqNoPrimaryTerm that is never mapped, so it is skipped", //
property.getFieldName(), entity.getType());
}
return;
}
buildPropertyMapping(propertiesNode, isRootObject, property);
} catch (IOException e) {
logger.warn("error mapping property with name {}", property.getName(), e);
}
});
}
InternalBuilder internalBuilder = new InternalBuilder();
return internalBuilder.buildPropertyMapping(entity, runtimeFields);
}
@Nullable
@ -259,315 +153,457 @@ public class MappingBuilder {
return null;
}
private void buildPropertyMapping(ObjectNode propertiesNode, boolean isRootObject,
ElasticsearchPersistentProperty property) throws IOException {
private class InternalBuilder {
if (property.isAnnotationPresent(Mapping.class)) {
private boolean writeTypeHints = true;
private List<String> excludeFromSource = new ArrayList<>();
private String nestedPropertyPrefix = "";
Mapping mapping = property.getRequiredAnnotation(Mapping.class);
protected String buildPropertyMapping(ElasticsearchPersistentEntity<?> entity,
@Nullable org.springframework.data.elasticsearch.core.document.Document runtimeFields) {
if (mapping.enabled()) {
String mappingPath = mapping.mappingPath();
try {
if (StringUtils.hasText(mappingPath)) {
writeTypeHints = entity.writeTypeHints();
ClassPathResource mappings = new ClassPathResource(mappingPath);
if (mappings.exists()) {
propertiesNode.putRawValue(property.getFieldName(),
new RawValue(StreamUtils.copyToString(mappings.getInputStream(), Charset.defaultCharset())));
return;
}
ObjectNode objectNode = objectMapper.createObjectNode();
// Dynamic templates
addDynamicTemplatesMapping(objectNode, entity);
mapEntity(objectNode, entity, true, "", false, FieldType.Auto, null,
entity.findAnnotation(DynamicMapping.class), runtimeFields);
if (!excludeFromSource.isEmpty()) {
ObjectNode sourceNode = objectNode.putObject(SOURCE);
ArrayNode excludes = sourceNode.putArray(SOURCE_EXCLUDES);
excludeFromSource.stream().map(TextNode::new).forEach(excludes::add);
}
} else {
applyDisabledPropertyMapping(propertiesNode, property);
return;
return objectMapper.writer().writeValueAsString(objectNode);
} catch (IOException e) {
throw new MappingException("could not build mapping", e);
}
}
if (property.isGeoPointProperty()) {
applyGeoPointFieldMapping(propertiesNode, property);
return;
}
private void writeTypeHintMapping(ObjectNode propertiesNode) throws IOException {
if (property.isGeoShapeProperty()) {
applyGeoShapeMapping(propertiesNode, property);
}
if (property.isJoinFieldProperty()) {
addJoinFieldMapping(propertiesNode, property);
}
Field fieldAnnotation = property.findAnnotation(Field.class);
boolean isCompletionProperty = property.isCompletionProperty();
boolean isNestedOrObjectProperty = isNestedOrObjectProperty(property);
DynamicMapping dynamicMapping = property.findAnnotation(DynamicMapping.class);
if (!isCompletionProperty && property.isEntity() && hasRelevantAnnotation(property)) {
if (fieldAnnotation == null) {
return;
}
if (isNestedOrObjectProperty) {
Iterator<? extends TypeInformation<?>> iterator = property.getPersistentEntityTypes().iterator();
ElasticsearchPersistentEntity<?> persistentEntity = iterator.hasNext()
? elasticsearchConverter.getMappingContext().getPersistentEntity(iterator.next())
: null;
mapEntity(propertiesNode, persistentEntity, false, property.getFieldName(), true, fieldAnnotation.type(),
fieldAnnotation, dynamicMapping, null);
return;
if (writeTypeHints) {
propertiesNode.set(TYPEHINT_PROPERTY, objectMapper.createObjectNode() //
.put(FIELD_PARAM_TYPE, TYPE_VALUE_KEYWORD) //
.put(FIELD_PARAM_INDEX, false) //
.put(FIELD_PARAM_DOC_VALUES, false));
}
}
MultiField multiField = property.findAnnotation(MultiField.class);
private void mapEntity(ObjectNode objectNode, @Nullable ElasticsearchPersistentEntity<?> entity,
boolean isRootObject, String nestedObjectFieldName, boolean nestedOrObjectField, FieldType fieldType,
@Nullable Field parentFieldAnnotation, @Nullable DynamicMapping dynamicMapping,
@Nullable Document runtimeFields) throws IOException {
if (isCompletionProperty) {
CompletionField completionField = property.findAnnotation(CompletionField.class);
applyCompletionFieldMapping(propertiesNode, property, completionField);
}
if (entity != null && entity.isAnnotationPresent(Mapping.class)) {
Mapping mappingAnnotation = entity.getRequiredAnnotation(Mapping.class);
if (isRootObject && fieldAnnotation != null && property.isIdProperty()) {
applyDefaultIdFieldMapping(propertiesNode, property);
} else if (multiField != null) {
addMultiFieldMapping(propertiesNode, property, multiField, isNestedOrObjectProperty, dynamicMapping);
} else if (fieldAnnotation != null) {
addSingleFieldMapping(propertiesNode, property, fieldAnnotation, isNestedOrObjectProperty, dynamicMapping);
}
}
if (!mappingAnnotation.enabled()) {
objectNode.put(MAPPING_ENABLED, false);
return;
}
private boolean hasRelevantAnnotation(ElasticsearchPersistentProperty property) {
if (mappingAnnotation.dateDetection() != Mapping.Detection.DEFAULT) {
objectNode.put(DATE_DETECTION, Boolean.parseBoolean(mappingAnnotation.dateDetection().name()));
}
return property.findAnnotation(Field.class) != null || property.findAnnotation(MultiField.class) != null
|| property.findAnnotation(GeoPointField.class) != null
|| property.findAnnotation(CompletionField.class) != null;
}
if (mappingAnnotation.numericDetection() != Mapping.Detection.DEFAULT) {
objectNode.put(NUMERIC_DETECTION, Boolean.parseBoolean(mappingAnnotation.numericDetection().name()));
}
private void applyGeoPointFieldMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property)
throws IOException {
propertiesNode.set(property.getFieldName(),
objectMapper.createObjectNode().put(FIELD_PARAM_TYPE, TYPE_VALUE_GEO_POINT));
}
if (mappingAnnotation.dynamicDateFormats().length > 0) {
objectNode.putArray(DYNAMIC_DATE_FORMATS).addAll(Arrays.stream(mappingAnnotation.dynamicDateFormats())
.map(TextNode::valueOf).collect(Collectors.toList()));
}
private void applyGeoShapeMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property)
throws IOException {
ObjectNode shapeNode = propertiesNode.putObject(property.getFieldName());
GeoShapeMappingParameters mappingParameters = GeoShapeMappingParameters
.from(property.findAnnotation(GeoShapeField.class));
mappingParameters.writeTypeAndParametersTo(shapeNode);
}
private void applyCompletionFieldMapping(ObjectNode propertyNode, ElasticsearchPersistentProperty property,
@Nullable CompletionField annotation) throws IOException {
ObjectNode completionNode = propertyNode.putObject(property.getFieldName());
completionNode.put(FIELD_PARAM_TYPE, TYPE_VALUE_COMPLETION);
if (annotation != null) {
completionNode.put(COMPLETION_MAX_INPUT_LENGTH, annotation.maxInputLength());
completionNode.put(COMPLETION_PRESERVE_POSITION_INCREMENTS, annotation.preservePositionIncrements());
completionNode.put(COMPLETION_PRESERVE_SEPARATORS, annotation.preserveSeparators());
if (StringUtils.hasLength(annotation.searchAnalyzer())) {
completionNode.put(FIELD_PARAM_SEARCH_ANALYZER, annotation.searchAnalyzer());
if (runtimeFields != null) {
objectNode.set(RUNTIME, objectMapper.convertValue(runtimeFields, JsonNode.class));
}
}
if (StringUtils.hasLength(annotation.analyzer())) {
completionNode.put(FIELD_PARAM_INDEX_ANALYZER, annotation.analyzer());
boolean writeNestedProperties = !isRootObject && (isAnyPropertyAnnotatedWithField(entity) || nestedOrObjectField);
if (writeNestedProperties) {
String type = nestedOrObjectField ? fieldType.toString().toLowerCase()
: FieldType.Object.toString().toLowerCase();
ObjectNode nestedObjectNode = objectMapper.createObjectNode();
nestedObjectNode.put(FIELD_PARAM_TYPE, type);
if (nestedOrObjectField && FieldType.Nested == fieldType && parentFieldAnnotation != null
&& parentFieldAnnotation.includeInParent()) {
nestedObjectNode.put(FIELD_INCLUDE_IN_PARENT, true);
}
objectNode.set(nestedObjectFieldName, nestedObjectNode);
// now go on with the nested one
objectNode = nestedObjectNode;
}
if (annotation.contexts().length > 0) {
if (entity != null && entity.dynamic() != Dynamic.INHERIT) {
objectNode.put(TYPE_DYNAMIC, entity.dynamic().name().toLowerCase());
} else if (dynamicMapping != null) {
objectNode.put(TYPE_DYNAMIC, dynamicMapping.value().name().toLowerCase());
}
ArrayNode contextsNode = completionNode.putArray(COMPLETION_CONTEXTS);
for (CompletionContext context : annotation.contexts()) {
ObjectNode propertiesNode = objectNode.putObject(FIELD_PROPERTIES);
ObjectNode contextNode = contextsNode.addObject();
contextNode.put(FIELD_CONTEXT_NAME, context.name());
contextNode.put(FIELD_CONTEXT_TYPE, context.type().name().toLowerCase());
writeTypeHintMapping(propertiesNode);
if (context.precision().length() > 0) {
contextNode.put(FIELD_CONTEXT_PRECISION, context.precision());
if (entity != null) {
entity.doWithProperties((PropertyHandler<ElasticsearchPersistentProperty>) property -> {
try {
if (property.isAnnotationPresent(Transient.class) || isInIgnoreFields(property, parentFieldAnnotation)) {
return;
}
if (property.isSeqNoPrimaryTermProperty()) {
if (property.isAnnotationPresent(Field.class)) {
logger.warn("Property {} of {} is annotated for inclusion in mapping, but its type is " + //
"SeqNoPrimaryTerm that is never mapped, so it is skipped", //
property.getFieldName(), entity.getType());
}
return;
}
buildPropertyMapping(propertiesNode, isRootObject, property);
} catch (IOException e) {
logger.warn("error mapping property with name {}", property.getName(), e);
}
});
}
}
if (StringUtils.hasText(context.path())) {
contextNode.put(FIELD_CONTEXT_PATH, context.path());
private void buildPropertyMapping(ObjectNode propertiesNode, boolean isRootObject,
ElasticsearchPersistentProperty property) throws IOException {
if (property.isAnnotationPresent(Mapping.class)) {
Mapping mapping = property.getRequiredAnnotation(Mapping.class);
if (mapping.enabled()) {
String mappingPath = mapping.mappingPath();
if (StringUtils.hasText(mappingPath)) {
ClassPathResource mappings = new ClassPathResource(mappingPath);
if (mappings.exists()) {
propertiesNode.putRawValue(property.getFieldName(),
new RawValue(StreamUtils.copyToString(mappings.getInputStream(), Charset.defaultCharset())));
return;
}
}
} else {
applyDisabledPropertyMapping(propertiesNode, property);
return;
}
}
if (property.isGeoPointProperty()) {
applyGeoPointFieldMapping(propertiesNode, property);
return;
}
if (property.isGeoShapeProperty()) {
applyGeoShapeMapping(propertiesNode, property);
}
if (property.isJoinFieldProperty()) {
addJoinFieldMapping(propertiesNode, property);
}
String nestedPropertyPath = nestedPropertyPrefix.isEmpty() ? property.getFieldName()
: nestedPropertyPrefix + '.' + property.getFieldName();
Field fieldAnnotation = property.findAnnotation(Field.class);
if (fieldAnnotation != null && fieldAnnotation.excludeFromSource()) {
excludeFromSource.add(nestedPropertyPath);
}
boolean isCompletionProperty = property.isCompletionProperty();
boolean isNestedOrObjectProperty = isNestedOrObjectProperty(property);
DynamicMapping dynamicMapping = property.findAnnotation(DynamicMapping.class);
if (!isCompletionProperty && property.isEntity() && hasRelevantAnnotation(property)) {
if (fieldAnnotation == null) {
return;
}
if (isNestedOrObjectProperty) {
Iterator<? extends TypeInformation<?>> iterator = property.getPersistentEntityTypes().iterator();
ElasticsearchPersistentEntity<?> persistentEntity = iterator.hasNext()
? elasticsearchConverter.getMappingContext().getPersistentEntity(iterator.next())
: null;
String currentNestedPropertyPrefix = nestedPropertyPrefix;
nestedPropertyPrefix = nestedPropertyPath;
mapEntity(propertiesNode, persistentEntity, false, property.getFieldName(), true, fieldAnnotation.type(),
fieldAnnotation, dynamicMapping, null);
nestedPropertyPrefix = currentNestedPropertyPrefix;
return;
}
}
MultiField multiField = property.findAnnotation(MultiField.class);
if (isCompletionProperty) {
CompletionField completionField = property.findAnnotation(CompletionField.class);
applyCompletionFieldMapping(propertiesNode, property, completionField);
}
if (isRootObject && fieldAnnotation != null && property.isIdProperty()) {
applyDefaultIdFieldMapping(propertiesNode, property);
} else if (multiField != null) {
addMultiFieldMapping(propertiesNode, property, multiField, isNestedOrObjectProperty, dynamicMapping);
} else if (fieldAnnotation != null) {
addSingleFieldMapping(propertiesNode, property, fieldAnnotation, isNestedOrObjectProperty, dynamicMapping);
}
}
private boolean hasRelevantAnnotation(ElasticsearchPersistentProperty property) {
return property.findAnnotation(Field.class) != null || property.findAnnotation(MultiField.class) != null
|| property.findAnnotation(GeoPointField.class) != null
|| property.findAnnotation(CompletionField.class) != null;
}
private void applyGeoPointFieldMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property)
throws IOException {
propertiesNode.set(property.getFieldName(),
objectMapper.createObjectNode().put(FIELD_PARAM_TYPE, TYPE_VALUE_GEO_POINT));
}
private void applyGeoShapeMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property)
throws IOException {
ObjectNode shapeNode = propertiesNode.putObject(property.getFieldName());
GeoShapeMappingParameters mappingParameters = GeoShapeMappingParameters
.from(property.findAnnotation(GeoShapeField.class));
mappingParameters.writeTypeAndParametersTo(shapeNode);
}
private void applyCompletionFieldMapping(ObjectNode propertyNode, ElasticsearchPersistentProperty property,
@Nullable CompletionField annotation) throws IOException {
ObjectNode completionNode = propertyNode.putObject(property.getFieldName());
completionNode.put(FIELD_PARAM_TYPE, TYPE_VALUE_COMPLETION);
if (annotation != null) {
completionNode.put(COMPLETION_MAX_INPUT_LENGTH, annotation.maxInputLength());
completionNode.put(COMPLETION_PRESERVE_POSITION_INCREMENTS, annotation.preservePositionIncrements());
completionNode.put(COMPLETION_PRESERVE_SEPARATORS, annotation.preserveSeparators());
if (StringUtils.hasLength(annotation.searchAnalyzer())) {
completionNode.put(FIELD_PARAM_SEARCH_ANALYZER, annotation.searchAnalyzer());
}
if (StringUtils.hasLength(annotation.analyzer())) {
completionNode.put(FIELD_PARAM_INDEX_ANALYZER, annotation.analyzer());
}
if (annotation.contexts().length > 0) {
ArrayNode contextsNode = completionNode.putArray(COMPLETION_CONTEXTS);
for (CompletionContext context : annotation.contexts()) {
ObjectNode contextNode = contextsNode.addObject();
contextNode.put(FIELD_CONTEXT_NAME, context.name());
contextNode.put(FIELD_CONTEXT_TYPE, context.type().name().toLowerCase());
if (context.precision().length() > 0) {
contextNode.put(FIELD_CONTEXT_PRECISION, context.precision());
}
if (StringUtils.hasText(context.path())) {
contextNode.put(FIELD_CONTEXT_PATH, context.path());
}
}
}
}
}
}
private void applyDefaultIdFieldMapping(ObjectNode propertyNode, ElasticsearchPersistentProperty property)
throws IOException {
propertyNode.set(property.getFieldName(), objectMapper.createObjectNode()//
.put(FIELD_PARAM_TYPE, TYPE_VALUE_KEYWORD) //
.put(FIELD_INDEX, true) //
);
}
private void applyDisabledPropertyMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property) {
try {
Field field = property.getRequiredAnnotation(Field.class);
if (field.type() != FieldType.Object) {
throw new IllegalArgumentException("Field type must be 'object");
}
propertiesNode.set(property.getFieldName(), objectMapper.createObjectNode() //
.put(FIELD_PARAM_TYPE, field.type().name().toLowerCase()) //
.put(MAPPING_ENABLED, false) //
private void applyDefaultIdFieldMapping(ObjectNode propertyNode, ElasticsearchPersistentProperty property)
throws IOException {
propertyNode.set(property.getFieldName(), objectMapper.createObjectNode()//
.put(FIELD_PARAM_TYPE, TYPE_VALUE_KEYWORD) //
.put(FIELD_INDEX, true) //
);
} catch (Exception e) {
throw new MappingException("Could not write enabled: false mapping for " + property.getFieldName(), e);
}
}
/**
* Add mapping for @Field annotation
*
* @throws IOException
*/
private void addSingleFieldMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property,
Field annotation, boolean nestedOrObjectField, @Nullable DynamicMapping dynamicMapping) throws IOException {
// build the property json, if empty skip it as this is no valid mapping
ObjectNode fieldNode = objectMapper.createObjectNode();
addFieldMappingParameters(fieldNode, annotation, nestedOrObjectField);
if (fieldNode.isEmpty()) {
return;
}
propertiesNode.set(property.getFieldName(), fieldNode);
private void applyDisabledPropertyMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property) {
if (nestedOrObjectField) {
if (annotation.dynamic() != Dynamic.INHERIT) {
fieldNode.put(TYPE_DYNAMIC, annotation.dynamic().name().toLowerCase());
} else if (dynamicMapping != null) {
fieldNode.put(TYPE_DYNAMIC, dynamicMapping.value().name().toLowerCase());
}
}
}
try {
Field field = property.getRequiredAnnotation(Field.class);
private void addJoinFieldMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property)
throws IOException {
JoinTypeRelation[] joinTypeRelations = property.getRequiredAnnotation(JoinTypeRelations.class).relations();
if (field.type() != FieldType.Object) {
throw new IllegalArgumentException("Field type must be 'object");
}
if (joinTypeRelations.length == 0) {
logger.warn("Property {}s type is JoinField but its annotation JoinTypeRelation is " + //
"not properly maintained", //
property.getFieldName());
return;
}
propertiesNode.set(property.getFieldName(), objectMapper.createObjectNode() //
.put(FIELD_PARAM_TYPE, field.type().name().toLowerCase()) //
.put(MAPPING_ENABLED, false) //
);
ObjectNode propertyNode = propertiesNode.putObject(property.getFieldName());
propertyNode.put(FIELD_PARAM_TYPE, TYPE_VALUE_JOIN);
ObjectNode relationsNode = propertyNode.putObject(JOIN_TYPE_RELATIONS);
for (JoinTypeRelation joinTypeRelation : joinTypeRelations) {
String parent = joinTypeRelation.parent();
String[] children = joinTypeRelation.children();
if (children.length > 1) {
relationsNode.putArray(parent)
.addAll(Arrays.stream(children).map(TextNode::valueOf).collect(Collectors.toList()));
} else if (children.length == 1) {
relationsNode.put(parent, children[0]);
}
}
}
/**
* Add mapping for @MultiField annotation
*
* @throws IOException
*/
private void addMultiFieldMapping(ObjectNode propertyNode, ElasticsearchPersistentProperty property,
MultiField annotation, boolean nestedOrObjectField, @Nullable DynamicMapping dynamicMapping) throws IOException {
// main field
ObjectNode mainFieldNode = objectMapper.createObjectNode();
propertyNode.set(property.getFieldName(), mainFieldNode);
if (nestedOrObjectField) {
if (annotation.mainField().dynamic() != Dynamic.INHERIT) {
mainFieldNode.put(TYPE_DYNAMIC, annotation.mainField().dynamic().name().toLowerCase());
} else if (dynamicMapping != null) {
mainFieldNode.put(TYPE_DYNAMIC, dynamicMapping.value().name().toLowerCase());
} catch (Exception e) {
throw new MappingException("Could not write enabled: false mapping for " + property.getFieldName(), e);
}
}
addFieldMappingParameters(mainFieldNode, annotation.mainField(), nestedOrObjectField);
/**
* Add mapping for @Field annotation
*
* @throws IOException
*/
private void addSingleFieldMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property,
Field annotation, boolean nestedOrObjectField, @Nullable DynamicMapping dynamicMapping) throws IOException {
// inner fields
ObjectNode innerFieldsNode = mainFieldNode.putObject("fields");
// build the property json, if empty skip it as this is no valid mapping
ObjectNode fieldNode = objectMapper.createObjectNode();
addFieldMappingParameters(fieldNode, annotation, nestedOrObjectField);
for (InnerField innerField : annotation.otherFields()) {
if (fieldNode.isEmpty()) {
return;
}
ObjectNode innerFieldNode = innerFieldsNode.putObject(innerField.suffix());
addFieldMappingParameters(innerFieldNode, innerField, false);
propertiesNode.set(property.getFieldName(), fieldNode);
if (nestedOrObjectField) {
if (annotation.dynamic() != Dynamic.INHERIT) {
fieldNode.put(TYPE_DYNAMIC, annotation.dynamic().name().toLowerCase());
} else if (dynamicMapping != null) {
fieldNode.put(TYPE_DYNAMIC, dynamicMapping.value().name().toLowerCase());
}
}
}
}
private void addFieldMappingParameters(ObjectNode fieldNode, Annotation annotation, boolean nestedOrObjectField)
throws IOException {
private void addJoinFieldMapping(ObjectNode propertiesNode, ElasticsearchPersistentProperty property)
throws IOException {
JoinTypeRelation[] joinTypeRelations = property.getRequiredAnnotation(JoinTypeRelations.class).relations();
MappingParameters mappingParameters = MappingParameters.from(annotation);
if (joinTypeRelations.length == 0) {
logger.warn("Property {}s type is JoinField but its annotation JoinTypeRelation is " + //
"not properly maintained", //
property.getFieldName());
return;
}
if (!nestedOrObjectField && mappingParameters.isStore()) {
fieldNode.put(FIELD_PARAM_STORE, true);
ObjectNode propertyNode = propertiesNode.putObject(property.getFieldName());
propertyNode.put(FIELD_PARAM_TYPE, TYPE_VALUE_JOIN);
ObjectNode relationsNode = propertyNode.putObject(JOIN_TYPE_RELATIONS);
for (JoinTypeRelation joinTypeRelation : joinTypeRelations) {
String parent = joinTypeRelation.parent();
String[] children = joinTypeRelation.children();
if (children.length > 1) {
relationsNode.putArray(parent)
.addAll(Arrays.stream(children).map(TextNode::valueOf).collect(Collectors.toList()));
} else if (children.length == 1) {
relationsNode.put(parent, children[0]);
}
}
}
mappingParameters.writeTypeAndParametersTo(fieldNode);
}
/**
* Apply mapping for dynamic templates.
*
* @throws IOException
*/
private void addDynamicTemplatesMapping(ObjectNode objectNode, ElasticsearchPersistentEntity<?> entity)
throws IOException {
/**
* Add mapping for @MultiField annotation
*
* @throws IOException
*/
private void addMultiFieldMapping(ObjectNode propertyNode, ElasticsearchPersistentProperty property,
MultiField annotation, boolean nestedOrObjectField, @Nullable DynamicMapping dynamicMapping)
throws IOException {
if (entity.isAnnotationPresent(DynamicTemplates.class)) {
String mappingPath = entity.getRequiredAnnotation(DynamicTemplates.class).mappingPath();
if (hasText(mappingPath)) {
// main field
ObjectNode mainFieldNode = objectMapper.createObjectNode();
propertyNode.set(property.getFieldName(), mainFieldNode);
String jsonString = ResourceUtil.readFileFromClasspath(mappingPath);
if (hasText(jsonString)) {
if (nestedOrObjectField) {
if (annotation.mainField().dynamic() != Dynamic.INHERIT) {
mainFieldNode.put(TYPE_DYNAMIC, annotation.mainField().dynamic().name().toLowerCase());
} else if (dynamicMapping != null) {
mainFieldNode.put(TYPE_DYNAMIC, dynamicMapping.value().name().toLowerCase());
}
}
JsonNode jsonNode = objectMapper.readTree(jsonString).get("dynamic_templates");
if (jsonNode != null && jsonNode.isArray()) {
objectNode.set(FIELD_DYNAMIC_TEMPLATES, jsonNode);
addFieldMappingParameters(mainFieldNode, annotation.mainField(), nestedOrObjectField);
// inner fields
ObjectNode innerFieldsNode = mainFieldNode.putObject("fields");
for (InnerField innerField : annotation.otherFields()) {
ObjectNode innerFieldNode = innerFieldsNode.putObject(innerField.suffix());
addFieldMappingParameters(innerFieldNode, innerField, false);
}
}
private void addFieldMappingParameters(ObjectNode fieldNode, Annotation annotation, boolean nestedOrObjectField)
throws IOException {
MappingParameters mappingParameters = MappingParameters.from(annotation);
if (!nestedOrObjectField && mappingParameters.isStore()) {
fieldNode.put(FIELD_PARAM_STORE, true);
}
mappingParameters.writeTypeAndParametersTo(fieldNode);
}
/**
* Apply mapping for dynamic templates.
*
* @throws IOException
*/
private void addDynamicTemplatesMapping(ObjectNode objectNode, ElasticsearchPersistentEntity<?> entity)
throws IOException {
if (entity.isAnnotationPresent(DynamicTemplates.class)) {
String mappingPath = entity.getRequiredAnnotation(DynamicTemplates.class).mappingPath();
if (hasText(mappingPath)) {
String jsonString = ResourceUtil.readFileFromClasspath(mappingPath);
if (hasText(jsonString)) {
JsonNode jsonNode = objectMapper.readTree(jsonString).get("dynamic_templates");
if (jsonNode != null && jsonNode.isArray()) {
objectNode.set(FIELD_DYNAMIC_TEMPLATES, jsonNode);
}
}
}
}
}
}
private boolean isAnyPropertyAnnotatedWithField(@Nullable ElasticsearchPersistentEntity entity) {
private boolean isAnyPropertyAnnotatedWithField(@Nullable ElasticsearchPersistentEntity entity) {
return entity != null && entity.getPersistentProperty(Field.class) != null;
}
private boolean isInIgnoreFields(ElasticsearchPersistentProperty property, @Nullable Field parentFieldAnnotation) {
if (null != parentFieldAnnotation) {
String[] ignoreFields = parentFieldAnnotation.ignoreFields();
return Arrays.asList(ignoreFields).contains(property.getFieldName());
return entity != null && entity.getPersistentProperty(Field.class) != null;
}
return false;
}
private boolean isNestedOrObjectProperty(ElasticsearchPersistentProperty property) {
private boolean isInIgnoreFields(ElasticsearchPersistentProperty property, @Nullable Field parentFieldAnnotation) {
Field fieldAnnotation = property.findAnnotation(Field.class);
return fieldAnnotation != null
&& (FieldType.Nested == fieldAnnotation.type() || FieldType.Object == fieldAnnotation.type());
if (null != parentFieldAnnotation) {
String[] ignoreFields = parentFieldAnnotation.ignoreFields();
return Arrays.asList(ignoreFields).contains(property.getFieldName());
}
return false;
}
private boolean isNestedOrObjectProperty(ElasticsearchPersistentProperty property) {
Field fieldAnnotation = property.findAnnotation(Field.class);
return fieldAnnotation != null
&& (FieldType.Nested == fieldAnnotation.type() || FieldType.Object == fieldAnnotation.type());
}
}
}

25
src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderIntegrationTests.java
View File

@ -26,6 +26,7 @@ import java.lang.Integer;
import java.lang.Object;
import java.math.BigDecimal;
import java.time.Instant;
import java.time.LocalDate;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
@ -47,13 +48,13 @@ import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
import org.springframework.data.elasticsearch.core.IndexOperations;
import org.springframework.data.elasticsearch.core.MappingContextBaseTests;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.data.elasticsearch.core.suggest.Completion;
import org.springframework.data.elasticsearch.core.geo.GeoPoint;
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
import org.springframework.data.elasticsearch.core.query.IndexQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.data.elasticsearch.core.query.SeqNoPrimaryTerm;
import org.springframework.data.elasticsearch.core.suggest.Completion;
import org.springframework.data.elasticsearch.junit.jupiter.ElasticsearchRestTemplateConfiguration;
import org.springframework.data.elasticsearch.junit.jupiter.SpringIntegrationTest;
import org.springframework.data.geo.Box;
@ -325,6 +326,16 @@ public class MappingBuilderIntegrationTests extends MappingContextBaseTests {
}
@Test // #796
@DisplayName("should write source excludes")
void shouldWriteSourceExcludes() {
IndexOperations indexOps = operations.indexOps(ExcludedFieldEntity.class);
indexOps.create();
indexOps.putMapping();
}
// region entities
@Document(indexName = "ignore-above-index")
static class IgnoreAboveEntity {
@ -1172,6 +1183,18 @@ public class MappingBuilderIntegrationTests extends MappingContextBaseTests {
@Field(type = Date, format = DateFormat.epoch_millis, name = "@timestamp") @Nullable private Instant timestamp;
}
@Document(indexName = "fields-excluded-from-source")
private static class ExcludedFieldEntity {
@Id @Nullable private String id;
@Nullable @Field(name = "excluded-date", type = Date, format = DateFormat.date,
excludeFromSource = true) private LocalDate excludedDate;
@Nullable @Field(type = Nested) private NestedExcludedFieldEntity nestedEntity;
}
private static class NestedExcludedFieldEntity {
@Nullable @Field(name = "excluded-text", type = Text, excludeFromSource = true) private String excludedText;
}
// endregion
}

57
src/test/java/org/springframework/data/elasticsearch/core/index/MappingBuilderUnitTests.java
View File

@ -42,10 +42,10 @@ import org.springframework.data.annotation.Id;
import org.springframework.data.annotation.Transient;
import org.springframework.data.elasticsearch.annotations.*;
import org.springframework.data.elasticsearch.core.MappingContextBaseTests;
import org.springframework.data.elasticsearch.core.suggest.Completion;
import org.springframework.data.elasticsearch.core.geo.GeoPoint;
import org.springframework.data.elasticsearch.core.mapping.SimpleElasticsearchMappingContext;
import org.springframework.data.elasticsearch.core.query.SeqNoPrimaryTerm;
import org.springframework.data.elasticsearch.core.suggest.Completion;
import org.springframework.data.geo.Box;
import org.springframework.data.geo.Circle;
import org.springframework.data.geo.Point;
@ -945,6 +945,49 @@ public class MappingBuilderUnitTests extends MappingContextBaseTests {
assertEquals(expected, mapping, true);
}
@Test // #796
@DisplayName("should add fields that are excluded from source")
void shouldAddFieldsThatAreExcludedFromSource() throws JSONException {
String expected = "{\n" + //
" \"properties\": {\n" + //
" \"_class\": {\n" + //
" \"type\": \"keyword\",\n" + //
" \"index\": false,\n" + //
" \"doc_values\": false\n" + //
" },\n" + //
" \"excluded-date\": {\n" + //
" \"type\": \"date\",\n" + //
" \"format\": \"date\"\n" + //
" },\n" + //
" \"nestedEntity\": {\n" + //
" \"type\": \"nested\",\n" + //
" \"properties\": {\n" + //
" \"_class\": {\n" + //
" \"type\": \"keyword\",\n" + //
" \"index\": false,\n" + //
" \"doc_values\": false\n" + //
" },\n" + //
" \"excluded-text\": {\n" + //
" \"type\": \"text\"\n" + //
" }\n" + //
" }\n" + //
" }\n" + //
" },\n" + //
" \"_source\": {\n" + //
" \"excludes\": [\n" + //
" \"excluded-date\",\n" + //
" \"nestedEntity.excluded-text\"\n" + //
" ]\n" + //
" }\n" + //
"}\n"; //
String mapping = getMappingBuilder().buildPropertyMapping(ExcludedFieldEntity.class);
assertEquals(expected, mapping, true);
}
// region entities
@Document(indexName = "ignore-above-index")
@ -1918,5 +1961,17 @@ public class MappingBuilderUnitTests extends MappingContextBaseTests {
@Id @Nullable private String id;
@Field(type = Date, format = DateFormat.epoch_millis, name = "@timestamp") @Nullable private Instant timestamp;
}
@Document(indexName = "fields-excluded-from-source")
private static class ExcludedFieldEntity {
@Id @Nullable private String id;
@Nullable @Field(name = "excluded-date", type = Date, format = DateFormat.date,
excludeFromSource = true) private LocalDate excludedDate;
@Nullable @Field(type = Nested) private NestedExcludedFieldEntity nestedEntity;
}
private static class NestedExcludedFieldEntity {
@Nullable @Field(name = "excluded-text", type = Text, excludeFromSource = true) private String excludedText;
}
// endregion
}