Fix reading response runtime field by mapping.

Original Pull Request #2432
Closes #2431

src/main/asciidoc/reference/elasticsearch-object-mapping.adoc

@@ -29,7 +29,7 @@ See <<elasticsearch.repositories.autocreation>>
 
 
 * `@Id`: Applied at the field level to mark the field used for identity purpose.
-* `@Transient`: By default all fields are mapped to the document when it is stored or retrieved, this annotation excludes the field.
+* `@Transient`, `@ReadOnlyProperty`, `@WriteOnlyProperty`: see the following section <<elasticsearch.mapping.meta-model.annotations.read-write>> for detailed information.
 * `@PersistenceConstructor`: Marks a given constructor - even a package protected one - to use when instantiating the object from the database.
 Constructor arguments are mapped by name to the key values in the retrieved Document.
 * `@Field`: Applied at the field level and defines properties of the field, most of the attributes map to the respective https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html[Elasticsearch Mapping] definitions (the following list is not complete, check the annotation Javadoc for a complete reference):
@@ -49,6 +49,25 @@ In difference to a registered Spring `Converter` this only converts the annotate
 
 The mapping metadata infrastructure is defined in a separate spring-data-commons project that is technology agnostic.
 
+[[elasticsearch.mapping.meta-model.annotations.read-write]]
+==== Controlling which properties are written to and read from Elasticsearch
+
+This section details the annotations that define if the value of a property is written to or 
+read from Elasticsearch.
+
+`@Transient`: A property annotated with this annotation will not be written to the mapping, it's value will not be 
+sent to Elasticsearch and when documents are returned from Elasticsearch, this property will not be set in the 
+resulting entity.
+
+`@ReadOnlyProperty`: A property with this annotaiton will not have its value written to Elasticsearch, but when 
+returning data, the proeprty will be filled with the value returned in the document from Elasticsearch.  One use case
+for this are runtime fields defined in the index mapping.
+
+`@WriteOnlyProperty`: A property with this annotaiton will have its value stored in Elasticsearch but will not be set
+with any value when reading document. This can be used for example for synthesized fields which should go into the 
+Elasticsearch index but are not used elsewhere.
+
+
 [[elasticsearch.mapping.meta-model.annotations.date-formats]]
 ==== Date format mapping
 

src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java

@@ -408,7 +408,7 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 			ElasticsearchPersistentProperty idProperty = persistentEntity.getIdProperty();
 
 			// Only deal with text because ES generated Ids are strings!
-			if (indexedObjectInformation.getId() != null && idProperty != null && idProperty.isWritable()
+			if (indexedObjectInformation.getId() != null && idProperty != null && idProperty.isReadable()
 					&& idProperty.getType().isAssignableFrom(String.class)) {
 				propertyAccessor.setProperty(idProperty, indexedObjectInformation.getId());
 			}

src/main/java/org/springframework/data/elasticsearch/core/AbstractReactiveElasticsearchTemplate.java

@@ -261,7 +261,7 @@ abstract public class AbstractReactiveElasticsearchTemplate
 			ElasticsearchPersistentProperty idProperty = persistentEntity.getIdProperty();
 
 			// Only deal with text because ES generated Ids are strings!
-			if (indexedObjectInformation.getId() != null && idProperty != null && idProperty.isWritable()
+			if (indexedObjectInformation.getId() != null && idProperty != null && idProperty.isReadable()
 					&& idProperty.getType().isAssignableFrom(String.class)) {
 				propertyAccessor.setProperty(idProperty, indexedObjectInformation.getId());
 			}

src/main/java/org/springframework/data/elasticsearch/core/convert/MappingElasticsearchConverter.java

@@ -349,7 +349,7 @@ public class MappingElasticsearchConverter
 					PersistentPropertyAccessor<R> propertyAccessor = new ConvertingPropertyAccessor<>(
 							targetEntity.getPropertyAccessor(result), conversionService);
 					// Only deal with String because ES generated Ids are strings !
-					if (idProperty != null && idProperty.isWritable() && idProperty.getType().isAssignableFrom(String.class)) {
+					if (idProperty != null && idProperty.isReadable() && idProperty.getType().isAssignableFrom(String.class)) {
 						propertyAccessor.setProperty(idProperty, document.getId());
 					}
 				}
@@ -411,7 +411,7 @@ public class MappingElasticsearchConverter
 
 			for (ElasticsearchPersistentProperty prop : entity) {
 
-				if (entity.isCreatorArgument(prop) || !prop.isReadable() || !prop.isWritable()) {
+				if (entity.isCreatorArgument(prop) || !prop.isReadable()) {
 					continue;
 				}
 

src/main/java/org/springframework/data/elasticsearch/core/mapping/ElasticsearchPersistentProperty.java

@@ -61,7 +61,7 @@ public interface ElasticsearchPersistentProperty extends PersistentProperty<Elas
 	PropertyValueConverter getPropertyValueConverter();
 
 	/**
-	 * Returns true if the property may be read.
+	 * Returns true if the property may be read from the store into the entity.
 	 *
 	 * @return true if readable, false otherwise
 	 * @since 4.0

src/main/java/org/springframework/data/elasticsearch/core/mapping/SimpleElasticsearchPersistentProperty.java

@@ -24,6 +24,7 @@ import java.util.List;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.springframework.beans.BeanUtils;
+import org.springframework.data.annotation.ReadOnlyProperty;
 import org.springframework.data.domain.Range;
 import org.springframework.data.elasticsearch.annotations.DateFormat;
 import org.springframework.data.elasticsearch.annotations.Field;
@@ -119,7 +120,7 @@ public class SimpleElasticsearchPersistentProperty extends
 
 	@Override
 	public boolean isWritable() {
-		return super.isWritable() && !isSeqNoPrimaryTermProperty();
+		return !isTransient() && !isSeqNoPrimaryTermProperty() && !isAnnotationPresent(ReadOnlyProperty.class);
 	}
 
 	@Override

src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchIntegrationTests.java

@@ -15,21 +15,20 @@
  */
 package org.springframework.data.elasticsearch.core;
 
-import static java.util.Collections.singletonList;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.assertj.core.api.Assertions.fail;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.springframework.data.elasticsearch.annotations.Document.VersionType.EXTERNAL_GTE;
+import static java.util.Collections.*;
+import static org.assertj.core.api.Assertions.*;
+import static org.elasticsearch.index.query.QueryBuilders.*;
+import static org.springframework.data.elasticsearch.annotations.Document.VersionType.*;
+import static org.springframework.data.elasticsearch.annotations.FieldType.*;
 import static org.springframework.data.elasticsearch.annotations.FieldType.Integer;
-import static org.springframework.data.elasticsearch.annotations.FieldType.Keyword;
-import static org.springframework.data.elasticsearch.annotations.FieldType.Text;
-import static org.springframework.data.elasticsearch.core.document.Document.create;
-import static org.springframework.data.elasticsearch.core.document.Document.parse;
-import static org.springframework.data.elasticsearch.utils.IdGenerator.nextIdAsString;
-import static org.springframework.data.elasticsearch.utils.IndexBuilder.buildIndex;
+import static org.springframework.data.elasticsearch.core.document.Document.*;
+import static org.springframework.data.elasticsearch.utils.IdGenerator.*;
+import static org.springframework.data.elasticsearch.utils.IndexBuilder.*;
 
+import java.lang.Double;
+import java.lang.Integer;
+import java.lang.Long;
+import java.lang.Object;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -55,7 +54,6 @@ import org.springframework.dao.InvalidDataAccessApiUsageException;
 import org.springframework.dao.OptimisticLockingFailureException;
 import org.springframework.data.annotation.AccessType;
 import org.springframework.data.annotation.Id;
-import org.springframework.data.annotation.ReadOnlyProperty;
 import org.springframework.data.annotation.Version;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Pageable;
@@ -79,20 +77,7 @@ import org.springframework.data.elasticsearch.core.index.AliasActions;
 import org.springframework.data.elasticsearch.core.index.Settings;
 import org.springframework.data.elasticsearch.core.join.JoinField;
 import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
-import org.springframework.data.elasticsearch.core.query.BaseQueryBuilder;
-import org.springframework.data.elasticsearch.core.query.Criteria;
-import org.springframework.data.elasticsearch.core.query.CriteriaQuery;
-import org.springframework.data.elasticsearch.core.query.FetchSourceFilterBuilder;
-import org.springframework.data.elasticsearch.core.query.HighlightQuery;
-import org.springframework.data.elasticsearch.core.query.IndexQuery;
-import org.springframework.data.elasticsearch.core.query.IndexQueryBuilder;
-import org.springframework.data.elasticsearch.core.query.IndicesOptions;
-import org.springframework.data.elasticsearch.core.query.MoreLikeThisQuery;
-import org.springframework.data.elasticsearch.core.query.Query;
-import org.springframework.data.elasticsearch.core.query.SeqNoPrimaryTerm;
-import org.springframework.data.elasticsearch.core.query.SourceFilter;
-import org.springframework.data.elasticsearch.core.query.StringQuery;
-import org.springframework.data.elasticsearch.core.query.UpdateQuery;
+import org.springframework.data.elasticsearch.core.query.*;
 import org.springframework.data.elasticsearch.core.query.highlight.Highlight;
 import org.springframework.data.elasticsearch.core.query.highlight.HighlightField;
 import org.springframework.data.elasticsearch.junit.jupiter.SpringIntegrationTest;
@@ -4582,7 +4567,7 @@ public abstract class ElasticsearchIntegrationTests {
 		@Field(type = FieldType.Keyword) private String part2;
 
 		@Id
-		@ReadOnlyProperty
+		@WriteOnlyProperty
 		@AccessType(AccessType.Type.PROPERTY)
 		public String getId() {
 			return part1 + '-' + part2;

src/test/java/org/springframework/data/elasticsearch/core/ReactiveElasticsearchIntegrationTests.java

@@ -52,7 +52,6 @@ import org.springframework.dao.DataAccessException;
 import org.springframework.dao.OptimisticLockingFailureException;
 import org.springframework.data.annotation.AccessType;
 import org.springframework.data.annotation.Id;
-import org.springframework.data.annotation.ReadOnlyProperty;
 import org.springframework.data.annotation.Version;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Pageable;
@@ -63,6 +62,7 @@ import org.springframework.data.elasticsearch.annotations.Field;
 import org.springframework.data.elasticsearch.annotations.FieldType;
 import org.springframework.data.elasticsearch.annotations.Mapping;
 import org.springframework.data.elasticsearch.annotations.Setting;
+import org.springframework.data.elasticsearch.annotations.WriteOnlyProperty;
 import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQuery;
 import org.springframework.data.elasticsearch.client.erhlc.NativeSearchQueryBuilder;
 import org.springframework.data.elasticsearch.client.erhlc.ReactiveElasticsearchTemplate;
@@ -1506,7 +1506,7 @@ public abstract class ReactiveElasticsearchIntegrationTests {
 		@Field(type = FieldType.Keyword) private String part2;
 
 		@Id
-		@ReadOnlyProperty
+		@WriteOnlyProperty
 		@AccessType(AccessType.Type.PROPERTY)
 		public String getId() {
 			return part1 + '-' + part2;

src/test/java/org/springframework/data/elasticsearch/core/RuntimeFieldsIntegrationTests.java

@@ -17,19 +17,24 @@ package org.springframework.data.elasticsearch.core;
 
 import static org.assertj.core.api.Assertions.*;
 
+import java.time.LocalDate;
+
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.annotation.Id;
-import org.springframework.data.elasticsearch.NewElasticsearchClientDevelopment;
+import org.springframework.data.annotation.ReadOnlyProperty;
+import org.springframework.data.elasticsearch.annotations.DateFormat;
 import org.springframework.data.elasticsearch.annotations.Document;
 import org.springframework.data.elasticsearch.annotations.Field;
 import org.springframework.data.elasticsearch.annotations.FieldType;
+import org.springframework.data.elasticsearch.annotations.Mapping;
 import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
 import org.springframework.data.elasticsearch.core.query.Criteria;
 import org.springframework.data.elasticsearch.core.query.CriteriaQuery;
+import org.springframework.data.elasticsearch.core.query.FetchSourceFilterBuilder;
 import org.springframework.data.elasticsearch.core.query.Query;
 import org.springframework.data.elasticsearch.junit.jupiter.SpringIntegrationTest;
 import org.springframework.data.elasticsearch.utils.IndexNameProvider;
@@ -44,14 +49,13 @@ public abstract class RuntimeFieldsIntegrationTests {
 
 	@Autowired private ElasticsearchOperations operations;
 	@Autowired protected IndexNameProvider indexNameProvider;
-	private IndexOperations indexOperations;
 
 	@BeforeEach
 	void setUp() {
 
 		indexNameProvider.increment();
-		indexOperations = operations.indexOps(SomethingToBuy.class);
-		indexOperations.createWithMapping();
+		operations.indexOps(SomethingToBuy.class).createWithMapping();
+		operations.indexOps(Person.class).createWithMapping();
 	}
 
 	@Test
@@ -91,6 +95,24 @@ public abstract class RuntimeFieldsIntegrationTests {
 		assertThat(searchHits.getSearchHit(0).getId()).isEqualTo("1");
 	}
 
+	@Test // #2431
+	@DisplayName("should return value from runtime field defined in mapping")
+	void shouldReturnValueFromRuntimeFieldDefinedInMapping() {
+
+		var person = new Person();
+		var years = 10;
+		person.setBirthDate(LocalDate.now().minusDays(years * 365 + 100));
+		operations.save(person);
+		var query = Query.findAll();
+		query.addFields("age");
+		query.addSourceFilter(new FetchSourceFilterBuilder().withIncludes("*").build());
+
+		var searchHits = operations.search(query, Person.class);
+
+		assertThat(searchHits.getTotalHits()).isEqualTo(1);
+		assertThat(searchHits.getSearchHit(0).getContent().getAge()).isEqualTo(years);
+	}
+
 	private void insert(String id, String description, double price) {
 		SomethingToBuy entity = new SomethingToBuy();
 		entity.setId(id);
@@ -99,7 +121,7 @@ public abstract class RuntimeFieldsIntegrationTests {
 		operations.save(entity);
 	}
 
-	@Document(indexName = "#{@indexNameProvider.indexName()}")
+	@Document(indexName = "#{@indexNameProvider.indexName()}-something")
 	private static class SomethingToBuy {
 		private @Id @Nullable String id;
 
@@ -136,4 +158,43 @@ public abstract class RuntimeFieldsIntegrationTests {
 			this.price = price;
 		}
 	}
+
+	@Document(indexName = "#{@indexNameProvider.indexName()}-person")
+	@Mapping(runtimeFieldsPath = "/runtime-fields-person.json")
+	public class Person {
+		@Nullable private String id;
+
+		@Field(type = FieldType.Date, format = DateFormat.basic_date)
+		@Nullable private LocalDate birthDate;
+
+		@ReadOnlyProperty // do not write to prevent ES from automapping
+		@Nullable private Integer age;
+
+		@Nullable
+		public String getId() {
+			return id;
+		}
+
+		public void setId(@Nullable String id) {
+			this.id = id;
+		}
+
+		@Nullable
+		public LocalDate getBirthDate() {
+			return birthDate;
+		}
+
+		public void setBirthDate(@Nullable LocalDate birthDate) {
+			this.birthDate = birthDate;
+		}
+
+		@Nullable
+		public Integer getAge() {
+			return age;
+		}
+
+		public void setAge(@Nullable Integer age) {
+			this.age = age;
+		}
+	}
 }

src/test/resources/runtime-fields-person.json

@@ -0,0 +1,9 @@
+{
+    "age": {
+        "type": "long",
+        "script": {
+            "lang": "painless",
+            "source": "Instant currentDate = Instant.ofEpochMilli(new Date().getTime()); Instant startDate = doc['birthDate'].value.toInstant(); emit(ChronoUnit.DAYS.between(startDate, currentDate) / 365);"
+        }
+    }
+}