From efd394370a169c276d33c595d9aabeab242431bc Mon Sep 17 00:00:00 2001 From: Peter-Josef Meisch Date: Fri, 30 Dec 2022 19:11:17 +0100 Subject: [PATCH] Implement search by template. Original Pull Request #2410 Closes #1891 --- .gitignore | 1 + .../reference/elasticsearch-misc.adoc | 114 ++++++++++++- .../reference/elasticsearch-operations.adoc | 6 + .../ResourceNotFoundException.java | 29 ++++ .../elc/ElasticsearchExceptionTranslator.java | 21 ++- .../client/elc/ElasticsearchTemplate.java | 53 ++++++ .../elc/ReactiveElasticsearchClient.java | 86 ++++++++++ .../elc/ReactiveElasticsearchTemplate.java | 60 ++++++- .../client/elc/RequestConverter.java | 89 +++++++++-- .../client/elc/ResponseConverter.java | 20 ++- .../elc/SearchDocumentResponseBuilder.java | 27 ++++ .../elasticsearch/client/elc/TypeUtils.java | 15 ++ .../erhlc/NativeSearchQueryBuilder.java | 9 +- .../core/AbstractElasticsearchTemplate.java | 24 +++ ...AbstractReactiveElasticsearchTemplate.java | 24 +++ .../core/ElasticsearchOperations.java | 3 +- .../core/ReactiveElasticsearchOperations.java | 4 +- .../elasticsearch/core/query/BaseQuery.java | 17 +- .../core/query/BaseQueryBuilder.java | 53 ++++-- .../core/query/IndicesOptions.java | 2 +- .../data/elasticsearch/core/query/Query.java | 14 +- .../core/query/SearchTemplateQuery.java | 62 +++++++ .../query/SearchTemplateQueryBuilder.java | 69 ++++++++ .../core/script/ReactiveScriptOperations.java | 50 ++++++ .../elasticsearch/core/script/Script.java | 74 +++++++++ .../core/script/ScriptOperations.java | 51 ++++++ .../core/script/package-info.java | 6 + ...tiveSearchTemplateELCIntegrationTests.java | 40 +++++ ...eactiveSearchTemplateIntegrationTests.java | 151 ++++++++++++++++++ .../SearchTemplateELCIntegrationTests.java | 39 +++++ .../core/SearchTemplateIntegrationTests.java | 145 +++++++++++++++++ 31 files changed, 1306 insertions(+), 52 deletions(-) create mode 100644 src/main/java/org/springframework/data/elasticsearch/ResourceNotFoundException.java create mode 100644 src/main/java/org/springframework/data/elasticsearch/core/query/SearchTemplateQuery.java create mode 100644 src/main/java/org/springframework/data/elasticsearch/core/query/SearchTemplateQueryBuilder.java create mode 100644 src/main/java/org/springframework/data/elasticsearch/core/script/ReactiveScriptOperations.java create mode 100644 src/main/java/org/springframework/data/elasticsearch/core/script/Script.java create mode 100644 src/main/java/org/springframework/data/elasticsearch/core/script/ScriptOperations.java create mode 100644 src/main/java/org/springframework/data/elasticsearch/core/script/package-info.java create mode 100644 src/test/java/org/springframework/data/elasticsearch/core/ReactiveSearchTemplateELCIntegrationTests.java create mode 100644 src/test/java/org/springframework/data/elasticsearch/core/ReactiveSearchTemplateIntegrationTests.java create mode 100644 src/test/java/org/springframework/data/elasticsearch/core/SearchTemplateELCIntegrationTests.java create mode 100644 src/test/java/org/springframework/data/elasticsearch/core/SearchTemplateIntegrationTests.java diff --git a/.gitignore b/.gitignore index ef50ee720..5f01412a0 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ target /zap.env +.localdocker-env diff --git a/src/main/asciidoc/reference/elasticsearch-misc.adoc b/src/main/asciidoc/reference/elasticsearch-misc.adoc index d78eb4b18..25bc8c5e7 100644 --- a/src/main/asciidoc/reference/elasticsearch-misc.adoc +++ b/src/main/asciidoc/reference/elasticsearch-misc.adoc @@ -17,7 +17,6 @@ The following arguments are available: * `refreshIntervall`, defaults to _"1s"_ * `indexStoreType`, defaults to _"fs"_ - It is as well possible to define https://www.elastic.co/guide/en/elasticsearch/reference/7.11/index-modules-index-sorting.html[index sorting] (check the linked Elasticsearch documentation for the possible field types and values): ==== @@ -133,9 +132,7 @@ stream.close(); ---- ==== -There are no methods in the `SearchOperations` API to access the scroll id, if it should be necessary to access this, -the following methods of the `AbstractElasticsearchTemplate` can be used (this is the base implementation for the -different `ElasticsearchOperations` implementations): +There are no methods in the `SearchOperations` API to access the scroll id, if it should be necessary to access this, the following methods of the `AbstractElasticsearchTemplate` can be used (this is the base implementation for the different `ElasticsearchOperations` implementations): ==== [source,java] @@ -281,7 +278,7 @@ This works with every implementation of the `Query` interface. [[elasticsearch.misc.point-in-time]] == Point In Time (PIT) API -`ElasticsearchOperations` supports the point in time API of Elasticsearch (see https://www.elastic.co/guide/en/elasticsearch/reference/8.3/point-in-time-api.html). +`ElasticsearchOperations` supports the point in time API of Elasticsearch (see https://www.elastic.co/guide/en/elasticsearch/reference/8.3/point-in-time-api.html). The following code snippet shows how to use this feature with a fictional `Person` class: ==== @@ -310,8 +307,115 @@ SearchHits searchHits2 = operations.search(query2, Person.class); operations.closePointInTime(searchHits2.getPointInTimeId()); <.> ---- + <.> create a point in time for an index (can be multiple names) and a keep-alive duration and retrieve its id <.> pass that id into the query to search together with the next keep-alive value <.> for the next query, use the id returned from the previous search <.> when done, close the point in time using the last returned id ==== + +[[elasticsearch.misc.searchtemplates]] +== Search Template support + +Use of the search template API is supported. +To use this, it first is necessary to create a stored script. +The `ElasticsearchOperations` interface extends `ScriptOperations` which provides the necessary functions. +The example used here assumes that we have `Person` entity with a property named `firstName`. +A search template script can be saved like this: + +==== +[source,java] +---- +import org.springframework.data.elasticsearch.core.ElasticsearchOperations; +import org.springframework.data.elasticsearch.core.script.Script; + +operations.putScript( <.> + Script.builder() + .withId("person-firstname") <.> + .withLanguage("mustache") <.> + .withSource(""" <.> + { + "query": { + "bool": { + "must": [ + { + "match": { + "firstName": "{{firstName}}" <.> + } + } + ] + } + }, + "from": "{{from}}", <.> + "size": "{{size}}" <.> + } + """) + .build() +); +---- + +<.> Use the `putScript()` method to store a search template script +<.> The name / id of the script +<.> Scripts that are used in search templates must be in the _mustache_ language. +<.> The script source +<.> The search parameter in the script +<.> Paging request offset +<.> Paging request size +==== + +To use a search template in a search query, Spring Data Elasticsearch provides the `SearchTemplateQuery`, an implementation of the `org.springframework.data.elasticsearch.core.query.Query` interface. + +In the following code, we will add a call using a search template query to a custom repository implementation (see +<>) as +an example how this can be integrated into a repository call. + +We first define the custom repository fragment interface: + +==== +[source,java] +---- +interface PersonCustomRepository { + SearchPage findByFirstNameWithSearchTemplate(String firstName, Pageable pageable); +} +---- +==== + +The implementation of this repository fragment looks like this: + +==== +[source,java] +---- +public class PersonCustomRepositoryImpl implements PersonCustomRepository { + + private final ElasticsearchOperations operations; + + public PersonCustomRepositoryImpl(ElasticsearchOperations operations) { + this.operations = operations; + } + + @Override + public SearchPage findByFirstNameWithSearchTemplate(String firstName, Pageable pageable) { + + var query = SearchTemplateQuery.builder() <.> + .withId("person-firstname") <.> + .withParams( + Map.of( <.> + "firstName", firstName, + "from", pageable.getOffset(), + "size", pageable.getPageSize() + ) + ) + .build(); + + SearchHits searchHits = operations.search(query, Person.class); <.> + + return SearchHitSupport.searchPageFor(searchHits, pageable); + } +} +---- + +<.> Create a `SearchTemplateQuery` +<.> Provide the id of the search template +<.> The parameters are passed in a `Map` +<.> Do the search in the same way as with the other query types. +==== diff --git a/src/main/asciidoc/reference/elasticsearch-operations.adoc b/src/main/asciidoc/reference/elasticsearch-operations.adoc index c170283e1..f49abda7c 100644 --- a/src/main/asciidoc/reference/elasticsearch-operations.adoc +++ b/src/main/asciidoc/reference/elasticsearch-operations.adoc @@ -234,3 +234,9 @@ Query query = NativeQuery.builder() SearchHits searchHits = operations.search(query, Person.class); ---- ==== + +[[elasticsearch.operations.searchtemplateScOp§query]] +=== SearchTemplateQuery + +This is a special implementation of the `Query` interface to be used in combination with a stored search template. +See <> for further information. diff --git a/src/main/java/org/springframework/data/elasticsearch/ResourceNotFoundException.java b/src/main/java/org/springframework/data/elasticsearch/ResourceNotFoundException.java new file mode 100644 index 000000000..db12bca4a --- /dev/null +++ b/src/main/java/org/springframework/data/elasticsearch/ResourceNotFoundException.java @@ -0,0 +1,29 @@ +/* + * Copyright 2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch; + +import org.springframework.dao.NonTransientDataAccessResourceException; + +/** + * @author Peter-Josef Meisch + * @since 5.1 + */ +public class ResourceNotFoundException extends NonTransientDataAccessResourceException { + + public ResourceNotFoundException(String msg) { + super(msg); + } +} diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchExceptionTranslator.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchExceptionTranslator.java index 8c8f9ac86..57298a223 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchExceptionTranslator.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchExceptionTranslator.java @@ -30,6 +30,7 @@ import org.springframework.dao.DataIntegrityViolationException; import org.springframework.dao.OptimisticLockingFailureException; import org.springframework.dao.support.PersistenceExceptionTranslator; import org.springframework.data.elasticsearch.NoSuchIndexException; +import org.springframework.data.elasticsearch.ResourceNotFoundException; import org.springframework.data.elasticsearch.UncategorizedElasticsearchException; /** @@ -77,16 +78,20 @@ public class ElasticsearchExceptionTranslator implements PersistenceExceptionTra var errorType = response.error().type(); var errorReason = response.error().reason() != null ? response.error().reason() : "undefined reason"; - if (response.status() == 404 && "index_not_found_exception".equals(errorType)) { + if (response.status() == 404) { - // noinspection RegExpRedundantEscape - Pattern pattern = Pattern.compile(".*no such index \\[(.*)\\]"); - String index = ""; - Matcher matcher = pattern.matcher(errorReason); - if (matcher.matches()) { - index = matcher.group(1); + if ("index_not_found_exception".equals(errorType)) { + // noinspection RegExpRedundantEscape + Pattern pattern = Pattern.compile(".*no such index \\[(.*)\\]"); + String index = ""; + Matcher matcher = pattern.matcher(errorReason); + if (matcher.matches()) { + index = matcher.group(1); + } + return new NoSuchIndexException(index); } - return new NoSuchIndexException(index); + + return new ResourceNotFoundException(errorReason); } String body = JsonUtils.toJson(response, jsonpMapper); diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java index bf3a83e8f..de479fed1 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ElasticsearchTemplate.java @@ -55,10 +55,12 @@ import org.springframework.data.elasticsearch.core.query.ByQueryResponse; import org.springframework.data.elasticsearch.core.query.IndexQuery; import org.springframework.data.elasticsearch.core.query.MoreLikeThisQuery; import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.core.query.SearchTemplateQuery; import org.springframework.data.elasticsearch.core.query.UpdateQuery; import org.springframework.data.elasticsearch.core.query.UpdateResponse; import org.springframework.data.elasticsearch.core.reindex.ReindexRequest; import org.springframework.data.elasticsearch.core.reindex.ReindexResponse; +import org.springframework.data.elasticsearch.core.script.Script; import org.springframework.lang.Nullable; import org.springframework.util.Assert; @@ -317,11 +319,21 @@ public class ElasticsearchTemplate extends AbstractElasticsearchTemplate { public SearchHits search(Query query, Class clazz, IndexCoordinates index) { Assert.notNull(query, "query must not be null"); + Assert.notNull(clazz, "clazz must not be null"); Assert.notNull(index, "index must not be null"); + if (query instanceof SearchTemplateQuery searchTemplateQuery) { + return doSearch(searchTemplateQuery, clazz, index); + } else { + return doSearch(query, clazz, index); + } + } + + protected SearchHits doSearch(Query query, Class clazz, IndexCoordinates index) { SearchRequest searchRequest = requestConverter.searchRequest(query, clazz, index, false); SearchResponse searchResponse = execute(client -> client.search(searchRequest, EntityAsMap.class)); + // noinspection DuplicatedCode ReadDocumentCallback readDocumentCallback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, index); SearchDocumentResponse.EntityCreator entityCreator = getEntityCreator(readDocumentCallback); SearchDocumentResponseCallback> callback = new ReadSearchDocumentResponseCallback<>(clazz, index); @@ -329,6 +341,18 @@ public class ElasticsearchTemplate extends AbstractElasticsearchTemplate { return callback.doWith(SearchDocumentResponseBuilder.from(searchResponse, entityCreator, jsonpMapper)); } + protected SearchHits doSearch(SearchTemplateQuery query, Class clazz, IndexCoordinates index) { + var searchTemplateRequest = requestConverter.searchTemplate(query, index); + var searchTemplateResponse = execute(client -> client.searchTemplate(searchTemplateRequest, EntityAsMap.class)); + + // noinspection DuplicatedCode + ReadDocumentCallback readDocumentCallback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, index); + SearchDocumentResponse.EntityCreator entityCreator = getEntityCreator(readDocumentCallback); + SearchDocumentResponseCallback> callback = new ReadSearchDocumentResponseCallback<>(clazz, index); + + return callback.doWith(SearchDocumentResponseBuilder.from(searchTemplateResponse, entityCreator, jsonpMapper)); + } + @Override protected SearchHits doSearch(MoreLikeThisQuery query, Class clazz, IndexCoordinates index) { @@ -513,6 +537,35 @@ public class ElasticsearchTemplate extends AbstractElasticsearchTemplate { // endregion + // region script methods + @Override + public boolean putScript(Script script) { + + Assert.notNull(script, "script must not be null"); + + var request = requestConverter.scriptPut(script); + return execute(client -> client.putScript(request)).acknowledged(); + } + + @Nullable + @Override + public Script getScript(String name) { + + Assert.notNull(name, "name must not be null"); + + var request = requestConverter.scriptGet(name); + return responseConverter.scriptResponse(execute(client -> client.getScript(request))); + } + + public boolean deleteScript(String name) { + + Assert.notNull(name, "name must not be null"); + + DeleteScriptRequest request = requestConverter.scriptDelete(name); + return execute(client -> client.deleteScript(request)).acknowledged(); + } + // endregion + // region client callback /** * Callback interface to be used with {@link #execute(ElasticsearchTemplate.ClientCallback)} for operating directly on diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchClient.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchClient.java index ebfb10fae..bcb919c99 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchClient.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchClient.java @@ -237,6 +237,29 @@ public class ReactiveElasticsearchClient extends ApiClient Mono> searchTemplate(SearchTemplateRequest request, Class tDocumentClass) { + + Assert.notNull(request, "request must not be null"); + Assert.notNull(tDocumentClass, "tDocumentClass must not be null"); + + return Mono.fromFuture(transport.performRequestAsync(request, + SearchTemplateRequest.createSearchTemplateEndpoint(this.getDeserializer(tDocumentClass)), transportOptions)); + } + + /** + * @since 5.1 + */ + public Mono> searchTemplate( + Function> fn, Class tDocumentClass) { + + Assert.notNull(fn, "fn must not be null"); + + return searchTemplate(fn.apply(new SearchTemplateRequest.Builder()).build(), tDocumentClass); + } + public Mono> scroll(ScrollRequest request, Class tDocumentClass) { Assert.notNull(request, "request must not be null"); @@ -320,4 +343,67 @@ public class ReactiveElasticsearchClient extends ApiClient putScript(PutScriptRequest request) { + + Assert.notNull(request, "request must not be null"); + + return Mono.fromFuture(transport.performRequestAsync(request, PutScriptRequest._ENDPOINT, transportOptions)); + } + + /** + * @since 5.1 + */ + public Mono putScript(Function> fn) { + + Assert.notNull(fn, "fn must not be null"); + + return putScript(fn.apply(new PutScriptRequest.Builder()).build()); + } + + /** + * @since 5.1 + */ + public Mono getScript(GetScriptRequest request) { + + Assert.notNull(request, "request must not be null"); + + return Mono.fromFuture(transport.performRequestAsync(request, GetScriptRequest._ENDPOINT, transportOptions)); + } + + /** + * @since 5.1 + */ + public Mono getScript(Function> fn) { + + Assert.notNull(fn, "fn must not be null"); + + return getScript(fn.apply(new GetScriptRequest.Builder()).build()); + } + + /** + * @since 5.1 + */ + public Mono deleteScript(DeleteScriptRequest request) { + + Assert.notNull(request, "request must not be null"); + + return Mono.fromFuture(transport.performRequestAsync(request, DeleteScriptRequest._ENDPOINT, transportOptions)); + } + + /** + * @since 5.1 + */ + public Mono deleteScript( + Function> fn) { + + Assert.notNull(fn, "fn must not be null"); + + return deleteScript(fn.apply(new DeleteScriptRequest.Builder()).build()); + } + // endregion + } diff --git a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java index 5954a9f03..a64190978 100644 --- a/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java +++ b/src/main/java/org/springframework/data/elasticsearch/client/elc/ReactiveElasticsearchTemplate.java @@ -62,10 +62,12 @@ import org.springframework.data.elasticsearch.core.query.BaseQuery; import org.springframework.data.elasticsearch.core.query.BulkOptions; import org.springframework.data.elasticsearch.core.query.ByQueryResponse; import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.core.query.SearchTemplateQuery; import org.springframework.data.elasticsearch.core.query.UpdateQuery; import org.springframework.data.elasticsearch.core.query.UpdateResponse; import org.springframework.data.elasticsearch.core.reindex.ReindexRequest; import org.springframework.data.elasticsearch.core.reindex.ReindexResponse; +import org.springframework.data.elasticsearch.core.script.Script; import org.springframework.lang.Nullable; import org.springframework.util.Assert; import org.springframework.util.CollectionUtils; @@ -341,12 +343,18 @@ public class ReactiveElasticsearchTemplate extends AbstractReactiveElasticsearch @Override protected Flux doFind(Query query, Class clazz, IndexCoordinates index) { - return Flux.defer(() -> { - boolean queryIsUnbounded = !(query.getPageable().isPaged() || query.isLimiting()); - - return queryIsUnbounded ? doFindUnbounded(query, clazz, index) : doFindBounded(query, clazz, index); - }); + Assert.notNull(query, "query must not be null"); + Assert.notNull(clazz, "clazz must not be null"); + Assert.notNull(index, "index must not be null"); + if (query instanceof SearchTemplateQuery searchTemplateQuery) { + return Flux.defer(() -> doSearch(searchTemplateQuery, clazz, index)); + } else { + return Flux.defer(() -> { + boolean queryIsUnbounded = !(query.getPageable().isPaged() || query.isLimiting()); + return queryIsUnbounded ? doFindUnbounded(query, clazz, index) : doFindBounded(query, clazz, index); + }); + } } private Flux doFindUnbounded(Query query, Class clazz, IndexCoordinates index) { @@ -465,6 +473,17 @@ public class ReactiveElasticsearchTemplate extends AbstractReactiveElasticsearch .map(entityAsMapHit -> DocumentAdapters.from(entityAsMapHit, jsonpMapper)); } + private Flux doSearch(SearchTemplateQuery query, Class clazz, IndexCoordinates index) { + + var request = requestConverter.searchTemplate(query, index); + + return Mono + .from(execute((ClientCallback>>) client -> client + .searchTemplate(request, EntityAsMap.class))) // + .flatMapIterable(entityAsMapSearchResponse -> entityAsMapSearchResponse.hits().hits()) // + .map(entityAsMapHit -> DocumentAdapters.from(entityAsMapHit, jsonpMapper)); + } + @Override protected Mono doFindForResponse(Query query, Class clazz, IndexCoordinates index) { @@ -519,6 +538,37 @@ public class ReactiveElasticsearchTemplate extends AbstractReactiveElasticsearch // endregion + // region script operations + @Override + public Mono putScript(Script script) { + + Assert.notNull(script, "script must not be null"); + + var request = requestConverter.scriptPut(script); + return Mono.from(execute((ClientCallback>) client -> client.putScript(request))) + .map(PutScriptResponse::acknowledged); + } + + @Override + public Mono