DATAES-750 - Migration guide and documentation update.

Original PR: #444

src/main/asciidoc/index.adoc

@@ -5,13 +5,14 @@ BioMed Central Development Team; Oliver Drotbohm; Greg Turnquist; Christoph Stro
 ifdef::backend-epub3[:front-cover-image: image:epub-cover.png[Front Cover,1050,1600]]
 :spring-data-commons-docs: ../../../../spring-data-commons/src/main/asciidoc
 
-(C) 2013-2019 The original author(s).
+(C) 2013-2020 The original author(s).
 
 NOTE: Copies of this document may be made for your own use and for distribution to others, provided that you do not charge any fee for such copies and further provided that each copy contains this Copyright Notice, whether distributed in print or electronically.
 
 toc::[]
 
 include::preface.adoc[]
+
 :leveloffset: +1
 include::{spring-data-commons-docs}/repositories.adoc[]
 :leveloffset: -1
@@ -23,9 +24,15 @@ include::{spring-data-commons-docs}/repositories.adoc[]
 include::reference/elasticsearch-clients.adoc[]
 include::reference/elasticsearch-object-mapping.adoc[]
 include::reference/elasticsearch-operations.adoc[]
+
 include::reference/elasticsearch-repositories.adoc[]
+
 include::{spring-data-commons-docs}/auditing.adoc[]
 include::reference/elasticsearch-auditing.adoc[]
+
+include::{spring-data-commons-docs}/entity-callbacks.adoc[]
+include::reference/elasticsearch-entity-callbacks.adoc[leveloffset=+1]
+
 include::reference/elasticsearch-misc.adoc[]
 :leveloffset: -1
 

src/main/asciidoc/preface.adoc

@@ -9,6 +9,7 @@ The Spring Data Elasticsearch project applies core Spring concepts to the develo
 You will notice similarities to the Spring data solr and mongodb support in the Spring Framework.
 
 include::reference/elasticsearch-new.adoc[leveloffset=+1]
+include::reference/elasticsearch-migration-guide-3.2-4.0.adoc[leveloffset=+1]
 
 [[preface.metadata]]
 == Project Metadata

src/main/asciidoc/reference/elasticsearch-entity-callbacks.adoc

@@ -0,0 +1,35 @@
+[[elasticsearch.entity-callbacks]]
+= Elasticsearch EntityCallbacks
+
+Spring Data Elasticsearch uses the `EntityCallback` API internally for its auditing support and reacts on the following callbacks:
+
+.Supported Entity Callbacks
+[%header,cols="4"]
+|===
+| Callback
+| Method
+| Description
+| Order
+
+| Reactive/BeforeConvertCallback
+| `onBeforeConvert(T entity, IndexCoordinates index)`
+| Invoked before a domain object is converted to `org.springframework.data.elasticsearch.core.document.Document`. Can return the `entity` or a modified entity which then will be converted.
+| `Ordered.LOWEST_PRECEDENCE`
+
+| Reactive/AfterConvertCallback
+| `onAfterConvert(T entity, Document document, IndexCoordinates indexCoordinates)`
+| Invoked after a domain object is converted from `org.springframework.data.elasticsearch.core.document.Document` on reading result data from Elasticsearch.
+| `Ordered.LOWEST_PRECEDENCE`
+
+| Reactive/AuditingEntityCallback
+| `onBeforeConvert(Object entity, IndexCoordinates index)`
+| Marks an auditable entity _created_ or _modified_
+| 100
+
+| Reactive/AfterSaveCallback
+| `T onAfterSave(T entity, IndexCoordinates index)`
+| Invoked after a domain object is saved.
+| `Ordered.LOWEST_PRECEDENCE`
+
+|===
+

src/main/asciidoc/reference/elasticsearch-migration-guide-3.2-4.0.adoc

@@ -0,0 +1,133 @@
+[[elasticsearch-migration-guide-3.2-4.0]]
+== Upgrading from 3.2.x to 4.0.x
+
+This section describes breaking changes from version 3.2.x to 4.0.x and how removed features can be replaced by new introduced features.
+
+
+=== Removal of the used Jackson Mapper.
+
+One of the changes in version 4.0.x is that Spring Data Elasticsearch does not use the Jackson Mapper anymore to map an entity to the JSON representation needed for Elasticsearch (see <<elasticsearch.mapping>>). In version 3.2.x the Jackson Mapper was the default that was used. It was possible to switch to the meta-model based converter (named `ElasticsearchEntityMapper`) by explicitly configuring it (<<elasticsearch.mapping.meta-model>>).
+
+In version 4.0.x the meta-model based converter is the only one that is available and does not need to be configured explicitly. If you had a custom configuration to enable the meta-model converter by providing a bean like this:
+
+[code,java]
+----
+@Bean
+@Override
+public EntityMapper entityMapper() {                                 
+
+  ElasticsearchEntityMapper entityMapper = new ElasticsearchEntityMapper(
+    elasticsearchMappingContext(), new DefaultConversionService()    
+  );
+  entityMapper.setConversions(elasticsearchCustomConversions());     
+
+  return entityMapper;
+}
+----
+
+You now have to remove this bean, the `ElasticsearchEntityMapper` interface has been removed.
+
+.Entity configuration
+Some users had custom Jackson annotations on the entity class, for example in order to define a custom name for the mapped document in Elasticsearch or to configure date conversions. These are not taken into account anymore. The needed functionality is now provided with Spring Data Elasticsearch's `@Field` annotation. Please see <<elasticsearch.mapping.meta-model.annotations>> for detailed information.
+
+
+=== Removal of implicit index name from query objects
+
+In 3.2.x the different query classes like `IndexQuery` or `SearchQuery` had properties that were taking the index name or index names that they were operating upon. If these were not set, the passed in entity was inspected to retrieve the index name that was set in the `@Document` annotation. +
+In 4.0.x the index name(s) must now be provided in an additional parameter of type `IndexCoordinates`. By separating this, it now is possible to use one query object against different indices.
+
+So for example the following code:
+
+[code,java]
+----
+IndexQuery indexQuery = new IndexQueryBuilder()
+  .withId(person.getId().toString())
+  .withObject(person)
+  .build();
+
+String documentId = elasticsearchOperations.index(indexQuery);
+----
+
+must be changed to:
+
+[code,java]
+----
+IndexCoordinates indexCoordinates = elasticsearchOperations.getIndexCoordinatesFor(person.getClass());
+
+IndexQuery indexQuery = new IndexQueryBuilder()
+  .withId(person.getId().toString())
+  .withObject(person)
+  .build();
+  
+String documentId = elasticsearchOperations.index(indexQuery, indexCoordinates);
+----
+
+To make it easier to work with entities and use the index name that is contained in the entitie's `@Document` annotation, new methods have been added like `DocumentOperations.save(T entity)`;
+
+
+=== The new Operations interfaces
+
+In version 3.2 there was the `ElasticsearchOperations` interface that defined all the methods for the `ElasticsearchTemplate` class. In version 4 the functions have been split into different interfaces, aligning these interfaces with the Elasticsearch API:
+
+* `DocumentOperations` are the functions related documents like saving, or deleting
+* `SearchOperations` contains the functions to search in Elasticsearch
+* `IndexOperations` define the functions to operate on indexes, like index creation or mappings creation.
+
+`ElasticsearchOperations` now extends `DocumentOperations` and `SearchOperations` and has methods get access to an `IndexOperations` instance.
+
+NOTE: All the functions from the `ElasticsearchOperations` interface in version 3.2 that are now moved to the `IndexOperations` interface are still available, they are marked as deprecated and have default implementations that delegate to the new implementation:
+
+[code,java]
+----
+/**
+ * Create an index for given indexName .
+ *
+ * @param indexName the name of the index
+ * @return {@literal true} if the index was created
+ * @deprecated since 4.0, use {@link IndexOperations#create()}
+ */
+@Deprecated
+default boolean createIndex(String indexName) {
+	return indexOps(IndexCoordinates.of(indexName)).create();
+}
+----
+
+
+=== Deprecations 
+
+==== Methods and classes
+
+Many functions and classes have been deprecated. These functions still work, but the Javadocs show with what they should be replaced.
+
+.Example from ElasticsearchOperations
+[code,java]
+----
+/**
+ * Retrieves an object from an index.
+ *
+ * @param query the query defining the id of the object to get
+ * @param clazz the type of the object to be returned
+ * @return the found object
+ * @deprecated since 4.0, use {@link #get(String, Class, IndexCoordinates)}
+ */
+@Deprecated
+@Nullable
+<T> T queryForObject(GetQuery query, Class<T> clazz);
+----    
+
+==== Elasticsearch deprecations
+
+Since version 7 the Elasticsearch `TransportClient` is deprecated, it will be removed with Elasticsearch version 8. Spring Data Elasticsearch deprecates the `ElasticsearchTemplate` class which uses the `TransportClient` in version 4.0.
+
+Mapping types were removed from Elasticsearch 7, they still exist as deprecated values in the Spring Data `@Document` annotation and the `IndexCoordinates` class but they are not used anymore internally.
+
+=== Removals
+
+* As already described, the `ElasticsearchEntityMapper` interface has been removed.
+
+* The `SearchQuery` interface has been merged into it's base interface `Query`, so it's occurrences can just be replaced with `Query`.
+
+* The method `org.springframework.data.elasticsearch.core.ElasticsearchOperations.query(SearchQuery query, ResultsExtractor<T> resultsExtractor);` and the `org.springframework.data.elasticsearch.core.ResultsExtractor` interface have been removed. These could be used to parse the result from Elasticsearch for cases in which the response mapping done with the Jackson based mapper was not enough. Since version 4.0, there are the new <<elasticsearch.operations.searchresulttypes>>  to return the information from an Elasticsearch response, so there is no need to expose this low level functionality.
+
+* The low level methods `startScroll`, `continueScroll` and `clearScroll` have been removed from the `ElasticsearchOperations` interface. For low level scroll API access, there now are `searchScrollStart`, `searchScrollContinue` and `searchScrollClear` methods on the `ElasticsearchRestTemplate` class.
+

src/main/asciidoc/reference/elasticsearch-new.adoc

@@ -11,8 +11,12 @@
 * Removal of the Jackson `ObjectMapper`, now using the <<elasticsearch.mapping.meta-model,MappingElasticsearchConverter>>
 * Cleanup of the API in the `*Operations` interfaces, grouping and renaming methods so that they match the Elasticsearch API, deprecating the old methods, aligning with other Spring Data modules.
 * Introduction of `SearchHit<T>` class to represent a found document together with the relevant result metadata for this document (i.e. _sortValues_).
-* Introduction of the `SearchHits<T>` class to represent a whole search result together with the metadata for the complete search result (i.e. _max_score_).
+* Introduction of the `SearchHits<T>` class to represent a whole search result together with the
+metadata for the complete search result (i.e. _max_score_).
+* Introduction of `SearchPage<T>` class to represent a paged result containing a `SearchHits<T>` instance.
 * Introduction of the `GeoDistanceOrder` class to be able to create sorting by geographical distance
+* Implementation of Auditing Support
+* Implementation of lifecycle entity callbacks
 
 [[new-features.3-2-0]]
 == New in Spring Data Elasticsearch 3.2

src/main/asciidoc/reference/elasticsearch-object-mapping.adoc

@@ -9,7 +9,7 @@ The main reasons for the removal of the Jackson based mapper are:
 
 * Custom mappings of fields needed to be done with annotations like `@JsonFormat` or `@JsonInclude`. This often caused problems when the same object was used in different JSON based datastores or sent over a JSON based API.
 * Custom field types and formats also need to be stored into the Elasticsearch index mappings. The Jackson based annotations did not fully provide all the information that is necessary to represent the types of Elasticsearch.
-* Fields must be mapped not only when converting fromand to entities, but also an query argument, returned data and on other places.
+* Fields must be mapped not only when converting from and to entities, but also in query argument, returned data and on other places.
 
 Using the `MappingElasticsearchConverter` now covers all these cases.
 
@@ -29,23 +29,23 @@ The following annotations are available:
 
 * `@Document`: Applied at the class level to indicate this class is a candidate for mapping to the database. The most important attributes are:
 ** `indexName`: the name of the index to store this entity in
-** `type`: the mapping type. If not set, the lowercased simple name of the class is used.
+** `type`: [line-through]#the mapping type. If not set, the lowercased simple name of the class is used.# (deprecated since version 4.0)
 ** `shards`: the number of shards for the index.
 ** `replicas`: the number of replicas for the index.
 ** `refreshIntervall`: Refresh interval for the index. Used for index creation. Default value is _"1s"_.
 ** `indexStoreType`:  Index storage type for the index. Used for index creation. Default value is _"fs"_.
 ** `createIndex`: Configuration whether to create an index on repository bootstrapping. Default value is _true_.
 ** `versionType`: Configuration of version management. Default value is _EXTERNAL_.
+
 * `@Id`: Applied at the field level to mark the field used for identity purpose.
-* `@Transient`: By default all private fields are mapped to the document, this annotation excludes the field where it is applied from being stored in the database
+* `@Transient`: By default all fields are mapped to the document when it is stored or retrieved, this annotation excludes the field.
 * `@PersistenceConstructor`: Marks a given constructor - even a package protected one - to use when instantiating the object from the database. Constructor arguments are mapped by name to the key values in the retrieved Document.
-* `@Field`: Applied at the field level and defines properties of the field, most of the attributes map to the respective https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html[Elasticsearch Mapping] definitions:
+* `@Field`: Applied at the field level and defines properties of the field, most of the attributes map to the respective https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html[Elasticsearch Mapping] definitions (the following list is not complete, check the annotation Javadoc for a complete reference):
 ** `name`: The name of the field as it will be represented in the Elasticsearch document, if not set, the Java field name is used.
-** `type`: the field type, can be one of _Text, Integer, Long, Date, Float, Double, Boolean, Object, Auto, Nested, Ip, Attachment, Keyword_. See https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html[Elasticsearch Mapping Types]
+** `type`: the field type, can be one of _Text, Keyword, Long, Integer, Short, Byte, Double, Float, Half_Float, Scaled_Float, Date, Date_Nanos, Boolean, Binary, Integer_Range, Float_Range, Long_Range, Double_Range, Date_Range, Ip_Range, Object, Nested, Ip, TokenCount, Percolator, Flattened, Search_As_You_Type_. See https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html[Elasticsearch Mapping Types]
 ** `format` and `pattern` custom definitions for the _Date_ type.
 ** `store`: Flag wether the original field value should be store in Elasticsearch, default value is _false_.
 ** `analyzer`, `searchAnalyzer`, `normalizer` for specifying custom custom analyzers and normalizer.
-** `copy_to`: the target field to copy multiple document fields to.
 * `@GeoPoint`: marks a field as _geo_point_ datatype. Can be omitted if the field is an instance of the `GeoPoint` class.
 
 The mapping metadata infrastructure is defined in a separate spring-data-commons project that is technology agnostic.

src/main/asciidoc/reference/elasticsearch-operations.adoc

@@ -125,7 +125,7 @@ include::reactive-elasticsearch-operations.adoc[leveloffset=+1]
 
 When a document is retrieved with the methods of the  `DocumentOperations` interface, just the found entity will be returned. When searching with the methods of the `SearchOperations` interface, additional information is available for each entity, for example the _score_ or the _sortValues_ of the found entity.
 
-In order to return this information, each entity is wrapped in a `SearchHit` object that contains this entity-specific additional information. These `SearchHit` objects themselves are returned within a `SearchHits` object which additionally contains informations about the whole search ike the _maxScore_ or requested aggregations.
+In order to return this information, each entity is wrapped in a `SearchHit` object that contains this entity-specific additional information. These `SearchHit` objects themselves are returned within a `SearchHits` object which additionally contains informations about the whole search like the _maxScore_ or requested aggregations. The following classes and interfaces are now available:
 
 .SearchHit<T>
 Contains the following information:
@@ -133,7 +133,7 @@ Contains the following information:
 * Id
 * Score
 * Sort Values
-* Highligth fields
+* Highlight fields
 * The retrieved entity of type <T>
 
 .SearchHits<T>
@@ -145,3 +145,12 @@ Contains the following information:
 * A list of `SearchHit<T>` objects
 * Returned aggregations
 
+.SearchPage<T>
+Defines a Spring Data `Page` that contains a `SearchHits<T>` element and can be used for paging access using repository methods.
+
+.SearchScrollHits<T>
+Returned by the low level scroll API functions in `ElasticsearchRestTemplate`, it enriches a `SearchHits<T>` with the Elasticsearch scroll id.
+
+.SearchHitsIterator<T>
+An Iterator returned by the streaming functions of the `SearchOperations` interface.
+

src/main/java/org/springframework/data/elasticsearch/annotations/FieldType.java

@@ -24,32 +24,32 @@ package org.springframework.data.elasticsearch.annotations;
  * @author Aleksei Arsenev
  */
 public enum FieldType {
-    Auto,
-    Text,
-    Keyword,
-    Long,
-    Integer,
-    Short,
-    Byte,
-    Double,
-    Float,
-    Half_Float,
-    Scaled_Float,
-    Date,
-    Date_Nanos,
-    Boolean,
-    Binary,
-    Integer_Range,
-    Float_Range,
-    Long_Range,
-    Double_Range,
-    Date_Range,
-    Ip_Range,
-    Object,
-    Nested,
-    Ip,
-    TokenCount,
-    Percolator,
-    Flattened,
-    Search_As_You_Type
+    Auto, //
+    Text, //
+    Keyword, //
+    Long, //
+    Integer, //
+    Short, //
+    Byte, //
+    Double, //
+    Float, //
+    Half_Float, //
+    Scaled_Float, //
+    Date, //
+    Date_Nanos, //
+    Boolean, //
+    Binary, //
+    Integer_Range, //
+    Float_Range, //
+    Long_Range, //
+    Double_Range, //
+    Date_Range, //
+    Ip_Range, //
+    Object, //
+    Nested, //
+    Ip, //
+    TokenCount, //
+    Percolator, //
+    Flattened, //
+    Search_As_You_Type //
 }

src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java

@@ -198,6 +198,12 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 		return get(query.getId(), clazz, index);
 	}
 
+	@Override
+	@Nullable
+	public <T> T queryForObject(GetQuery query, Class<T> clazz) {
+		return get(query.getId(), clazz, getIndexCoordinatesFor(clazz));
+	}
+
 	@Override
 	public boolean exists(String id, Class<?> clazz) {
 		return exists(id, getIndexCoordinatesFor(clazz));

src/main/java/org/springframework/data/elasticsearch/core/DocumentOperations.java

@@ -256,5 +256,16 @@ public interface DocumentOperations {
 	@Nullable
 	<T> T get(GetQuery query, Class<T> clazz, IndexCoordinates index);
 
+	/**
+	 * Retrieves an object from an index.
+	 *
+	 * @param query the query defining the id of the object to get
+	 * @param clazz the type of the object to be returned
+	 * @return the found object
+	 * @deprecated since 4.0, use {@link #get(String, Class, IndexCoordinates)}
+	 */
+	@Deprecated
+	@Nullable
+	<T> T queryForObject(GetQuery query, Class<T> clazz);
 	// endregion
 }

src/main/java/org/springframework/data/elasticsearch/core/ElasticsearchOperations.java

@@ -60,17 +60,17 @@ public interface ElasticsearchOperations extends DocumentOperations, SearchOpera
 	IndexCoordinates getIndexCoordinatesFor(Class<?> clazz);
 
 	// region IndexOperations
-	/**
-	 * Create an index for given indexName if it does not already exist.
-	 *
-	 * @param indexName the name of the index
-	 * @return {@literal true} if the index was created
-	 * @deprecated since 4.0, use {@link IndexOperations#create()}
-	 */
-	@Deprecated
-	default boolean createIndex(String indexName) {
-		return indexOps(IndexCoordinates.of(indexName)).create();
-	}
+/**
+ * Create an index for given indexName .
+ *
+ * @param indexName the name of the index
+ * @return {@literal true} if the index was created
+ * @deprecated since 4.0, use {@link IndexOperations#create()}
+ */
+@Deprecated
+default boolean createIndex(String indexName) {
+	return indexOps(IndexCoordinates.of(indexName)).create();
+}
 
 	/**
 	 * Create an index for given indexName and Settings.