Parse returned suggest data using the new client.

Original Pull Request #2187
Closes #2154

src/main/java/org/springframework/data/elasticsearch/client/elc/DocumentAdapters.java

@@ -19,6 +19,7 @@ import co.elastic.clients.elasticsearch.core.GetResponse;
 import co.elastic.clients.elasticsearch.core.MgetResponse;
 import co.elastic.clients.elasticsearch.core.explain.ExplanationDetail;
 import co.elastic.clients.elasticsearch.core.get.GetResult;
+import co.elastic.clients.elasticsearch.core.search.CompletionSuggestOption;
 import co.elastic.clients.elasticsearch.core.search.Hit;
 import co.elastic.clients.elasticsearch.core.search.NestedIdentity;
 import co.elastic.clients.json.JsonData;
@@ -142,6 +143,21 @@ final class DocumentAdapters {
 				highlightFields, innerHits, nestedMetaData, explanation, matchedQueries, hit.routing());
 	}
 
+	public static SearchDocument from(CompletionSuggestOption<EntityAsMap> completionSuggestOption) {
+
+		Document document = completionSuggestOption.source() != null ? Document.from(completionSuggestOption.source())
+				: Document.create();
+		document.setIndex(completionSuggestOption.index());
+
+		if (completionSuggestOption.id() != null) {
+			document.setId(completionSuggestOption.id());
+		}
+
+		float score = completionSuggestOption.score() != null ? completionSuggestOption.score().floatValue() : Float.NaN;
+		return new SearchDocumentAdapter(document, score, new Object[] {}, Collections.emptyMap(), Collections.emptyMap(),
+				Collections.emptyMap(), null, null, null, completionSuggestOption.routing());
+	}
+
 	@Nullable
 	private static Explanation from(@Nullable co.elastic.clients.elasticsearch.core.explain.Explanation explanation) {
 

src/main/java/org/springframework/data/elasticsearch/client/elc/SearchDocumentResponseBuilder.java

@@ -17,24 +17,30 @@ package org.springframework.data.elasticsearch.client.elc;
 
 import co.elastic.clients.elasticsearch._types.aggregations.Aggregate;
 import co.elastic.clients.elasticsearch.core.SearchResponse;
-import co.elastic.clients.elasticsearch.core.search.Hit;
+import co.elastic.clients.elasticsearch.core.search.*;
-import co.elastic.clients.elasticsearch.core.search.HitsMetadata;
-import co.elastic.clients.elasticsearch.core.search.ResponseBody;
-import co.elastic.clients.elasticsearch.core.search.Suggestion;
-import co.elastic.clients.elasticsearch.core.search.TotalHits;
 import co.elastic.clients.json.JsonpMapper;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.elasticsearch.search.SearchHits;
 import org.springframework.data.elasticsearch.core.TotalHitsRelation;
 import org.springframework.data.elasticsearch.core.document.SearchDocument;
 import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse;
+import org.springframework.data.elasticsearch.core.suggest.response.CompletionSuggestion;
+import org.springframework.data.elasticsearch.core.suggest.response.PhraseSuggestion;
 import org.springframework.data.elasticsearch.core.suggest.response.Suggest;
+import org.springframework.data.elasticsearch.core.suggest.response.TermSuggestion;
+import org.springframework.data.elasticsearch.support.ScoreDoc;
 import org.springframework.lang.Nullable;
 import org.springframework.util.Assert;
+import org.springframework.util.CollectionUtils;
 
 /**
  * Factory class to create {@link SearchDocumentResponse} instances.
@@ -43,6 +49,9 @@ import org.springframework.util.Assert;
  * @since 4.4
  */
 class SearchDocumentResponseBuilder {
+
+	private static final Log LOGGER = LogFactory.getLog(SearchDocumentResponseBuilder.class);
+
 	/**
 	 * creates a SearchDocumentResponse from the {@link SearchResponse}
 	 *
@@ -80,7 +89,7 @@ class SearchDocumentResponseBuilder {
 	 * @return the {@link SearchDocumentResponse}
 	 */
 	public static <T> SearchDocumentResponse from(HitsMetadata<?> hitsMetadata, @Nullable String scrollId,
-			Map<String, Aggregate> aggregations, Map<String, List<Suggestion<EntityAsMap>>> suggestES,
+			@Nullable Map<String, Aggregate> aggregations, Map<String, List<Suggestion<EntityAsMap>>> suggestES,
 			SearchDocumentResponse.EntityCreator<T> entityCreator, JsonpMapper jsonpMapper) {
 
 		Assert.notNull(hitsMetadata, "hitsMetadata must not be null");
@@ -116,10 +125,116 @@ class SearchDocumentResponseBuilder {
 		ElasticsearchAggregations aggregationsContainer = aggregations != null ? new ElasticsearchAggregations(aggregations)
 				: null;
 
-		// todo #2154
+		Suggest suggest = suggestFrom(suggestES, entityCreator);
-		Suggest suggest = null;
 
 		return new SearchDocumentResponse(totalHits, totalHitsRelation, maxScore, scrollId, searchDocuments,
 				aggregationsContainer, suggest);
 	}
+
+	@Nullable
+	private static <T> Suggest suggestFrom(Map<String, List<Suggestion<EntityAsMap>>> suggestES,
+			SearchDocumentResponse.EntityCreator<T> entityCreator) {
+
+		if (CollectionUtils.isEmpty(suggestES)) {
+			return null;
+		}
+
+		List<Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>> suggestions = new ArrayList<>();
+
+		suggestES.forEach((name, suggestionsES) -> {
+
+			if (!suggestionsES.isEmpty()) {
+				// take the type from the first entry
+				switch (suggestionsES.get(0)._kind()) {
+					case Term: {
+						suggestions.add(getTermSuggestion(name, suggestionsES));
+						break;
+					}
+					case Phrase: {
+						suggestions.add(getPhraseSuggestion(name, suggestionsES));
+						break;
+					}
+					case Completion: {
+						suggestions.add(getCompletionSuggestion(name, suggestionsES, entityCreator));
+						break;
+					}
+					default:
+						break;
+				}
+			}
+		});
+
+		// todo: hasScoreDocs checks if any one
+		boolean hasScoreDocs = false;
+
+		return new Suggest(suggestions, hasScoreDocs);
+	}
+
+	private static TermSuggestion getTermSuggestion(String name, List<Suggestion<EntityAsMap>> suggestionsES) {
+
+		List<TermSuggestion.Entry> entries = new ArrayList<>();
+		suggestionsES.forEach(suggestionES -> {
+			TermSuggest termSuggest = suggestionES.term();
+
+			TermSuggestOption optionES = termSuggest.options();
+			List<TermSuggestion.Entry.Option> options = new ArrayList<>();
+			options.add(new TermSuggestion.Entry.Option(optionES.text(), null, optionES.score(), null,
+					Math.toIntExact(optionES.freq())));
+			entries.add(new TermSuggestion.Entry(termSuggest.text(), termSuggest.offset(), termSuggest.length(), options));
+		});
+		return new TermSuggestion(name, suggestionsES.size(), entries, null);
+	}
+
+	private static PhraseSuggestion getPhraseSuggestion(String name, List<Suggestion<EntityAsMap>> suggestionsES) {
+
+		List<PhraseSuggestion.Entry> entries = new ArrayList<>();
+		suggestionsES.forEach(suggestionES -> {
+			PhraseSuggest phraseSuggest = suggestionES.phrase();
+			PhraseSuggestOption optionES = phraseSuggest.options();
+			List<PhraseSuggestion.Entry.Option> options = new ArrayList<>();
+			options.add(new PhraseSuggestion.Entry.Option(optionES.text(), optionES.highlighted(), null, null));
+			entries.add(new PhraseSuggestion.Entry(phraseSuggest.text(), phraseSuggest.offset(), phraseSuggest.length(),
+					options, null));
+		});
+		return new PhraseSuggestion(name, suggestionsES.size(), entries);
+	}
+
+	private static <T> CompletionSuggestion<T> getCompletionSuggestion(String name,
+			List<Suggestion<EntityAsMap>> suggestionsES, SearchDocumentResponse.EntityCreator<T> entityCreator) {
+		List<CompletionSuggestion.Entry<T>> entries = new ArrayList<>();
+		suggestionsES.forEach(suggestionES -> {
+			CompletionSuggest<EntityAsMap> completionSuggest = suggestionES.completion();
+			List<CompletionSuggestion.Entry.Option<T>> options = new ArrayList<>();
+			List<CompletionSuggestOption<EntityAsMap>> optionsES = completionSuggest.options();
+			optionsES.forEach(optionES -> {
+				SearchDocument searchDocument = (optionES.source() != null) ? DocumentAdapters.from(optionES) : null;
+				T hitEntity = null;
+
+				if (searchDocument != null) {
+					try {
+						hitEntity = entityCreator.apply(searchDocument).get();
+					} catch (Exception e) {
+						if (LOGGER.isWarnEnabled()) {
+							LOGGER.warn("Error creating entity from SearchDocument: " + e.getMessage());
+						}
+					}
+				}
+
+				Map<String, Set<String>> contexts = new HashMap<>();
+				optionES.contexts().forEach((key, contextList) -> contexts.put(key,
+						contextList.stream().map(context -> context._get().toString()).collect(Collectors.toSet())));
+
+				// response from the new client does not have a doc and shardindex as the ScoreDoc from the old client responses
+
+				options.add(new CompletionSuggestion.Entry.Option<>(optionES.text(), null, optionES.score(),
+						optionES.collateMatch() != null ? optionES.collateMatch() : false, contexts,
+						new ScoreDoc(optionES.score() != null ? optionES.score() : Double.NaN, null, null), searchDocument,
+						hitEntity));
+			});
+
+			entries.add(new CompletionSuggestion.Entry<>(completionSuggest.text(), completionSuggest.offset(),
+					completionSuggest.length(), options));
+		});
+		return new CompletionSuggestion<>(name, suggestionsES.size(), entries);
+	}
 }

src/main/java/org/springframework/data/elasticsearch/core/AbstractElasticsearchTemplate.java

@@ -745,9 +745,9 @@ public abstract class AbstractElasticsearchTemplate implements ElasticsearchOper
 
 			IndexedObjectInformation indexedObjectInformation = IndexedObjectInformation.of( //
 					documentAfterLoad.hasId() ? documentAfterLoad.getId() : null, //
-					documentAfterLoad.getSeqNo(), //
+					documentAfterLoad.hasSeqNo() ? documentAfterLoad.getSeqNo() : null, //
-					documentAfterLoad.getPrimaryTerm(), //
+					documentAfterLoad.hasPrimaryTerm() ? documentAfterLoad.getPrimaryTerm() : null, //
-					documentAfterLoad.getVersion()); //
+					documentAfterLoad.hasVersion() ? documentAfterLoad.getVersion() : null); //
 			entity = updateIndexedObject(entity, indexedObjectInformation);
 
 			return maybeCallbackAfterConvert(entity, documentAfterLoad, index);

src/main/java/org/springframework/data/elasticsearch/core/AbstractReactiveElasticsearchTemplate.java

@@ -549,9 +549,9 @@ abstract public class AbstractReactiveElasticsearchTemplate
 
 						IndexedObjectInformation indexedObjectInformation = IndexedObjectInformation.of( //
 								documentAfterLoad.hasId() ? documentAfterLoad.getId() : null, //
-								documentAfterLoad.getSeqNo(), //
+								documentAfterLoad.hasSeqNo() ? documentAfterLoad.getSeqNo() : null, //
-								documentAfterLoad.getPrimaryTerm(), //
+								documentAfterLoad.hasPrimaryTerm() ? documentAfterLoad.getPrimaryTerm() : null, //
-								documentAfterLoad.getVersion()); //
+								documentAfterLoad.hasVersion() ? documentAfterLoad.getVersion() : null); //
 						entity = updateIndexedObject(entity, indexedObjectInformation);
 
 						return maybeCallAfterConvert(entity, documentAfterLoad, index);

src/main/java/org/springframework/data/elasticsearch/core/document/SearchDocumentResponseBuilder.java

@@ -156,7 +156,7 @@ public class SearchDocumentResponseBuilder {
 					List<PhraseSuggestion.Entry.Option> options = new ArrayList<>();
 					for (org.elasticsearch.search.suggest.phrase.PhraseSuggestion.Entry.Option optionES : entryES) {
 						options.add(new PhraseSuggestion.Entry.Option(textToString(optionES.getText()),
-								textToString(optionES.getHighlighted()), optionES.getScore(), optionES.collateMatch()));
+								textToString(optionES.getHighlighted()), (double) optionES.getScore(), optionES.collateMatch()));
 					}
 
 					entries.add(new PhraseSuggestion.Entry(textToString(entryES.getText()), entryES.getOffset(),
@@ -188,7 +188,7 @@ public class SearchDocumentResponseBuilder {
 						}
 
 						options.add(new CompletionSuggestion.Entry.Option<>(textToString(optionES.getText()),
-								textToString(optionES.getHighlighted()), optionES.getScore(), optionES.collateMatch(),
+								textToString(optionES.getHighlighted()), (double) optionES.getScore(), optionES.collateMatch(),
 								optionES.getContexts(), scoreDocFrom(optionES.getDoc()), searchDocument, hitEntity));
 					}
 

src/main/java/org/springframework/data/elasticsearch/core/suggest/response/CompletionSuggestion.java

@@ -49,7 +49,7 @@ public class CompletionSuggestion<T> extends Suggest.Suggestion<CompletionSugges
 			@Nullable private final T hitEntity;
 			@Nullable private SearchHit<T> searchHit;
 
-			public Option(String text, String highlighted, float score, Boolean collateMatch,
+			public Option(String text, @Nullable String highlighted, @Nullable Double score, Boolean collateMatch,
 					Map<String, Set<String>> contexts, ScoreDoc scoreDoc, @Nullable SearchDocument searchDocument,
 					@Nullable T hitEntity) {
 				super(text, highlighted, score, collateMatch);

src/main/java/org/springframework/data/elasticsearch/core/suggest/response/PhraseSuggestion.java

@@ -15,6 +15,8 @@
  */
 package org.springframework.data.elasticsearch.core.suggest.response;
 
+import org.springframework.lang.Nullable;
+
 import java.util.List;
 
 /**
@@ -29,20 +31,21 @@ public class PhraseSuggestion extends Suggest.Suggestion<PhraseSuggestion.Entry>
 
 	public static class Entry extends Suggest.Suggestion.Entry<Entry.Option> {
 
-		private final double cutoffScore;
+		@Nullable private final Double cutoffScore;
 
-		public Entry(String text, int offset, int length, List<Option> options, double cutoffScore) {
+		public Entry(String text, int offset, int length, List<Option> options, @Nullable Double cutoffScore) {
 			super(text, offset, length, options);
 			this.cutoffScore = cutoffScore;
 		}
 
-		public double getCutoffScore() {
+		@Nullable
+		public Double getCutoffScore() {
 			return cutoffScore;
 		}
 
 		public static class Option extends Suggest.Suggestion.Entry.Option {
 
-			public Option(String text, String highlighted, float score, Boolean collateMatch) {
+			public Option(String text, String highlighted, @Nullable Double score, @Nullable Boolean collateMatch) {
 				super(text, highlighted, score, collateMatch);
 			}
 		}

src/main/java/org/springframework/data/elasticsearch/core/suggest/response/Suggest.java

@@ -106,11 +106,12 @@ public class Suggest {
 
 			public abstract static class Option {
 				private final String text;
-				private final String highlighted;
+				@Nullable private final String highlighted;
-				private final float score;
+				@Nullable private final Double score;
 				@Nullable private final Boolean collateMatch;
 
-				public Option(String text, String highlighted, float score, @Nullable Boolean collateMatch) {
+				public Option(String text, @Nullable String highlighted, @Nullable Double score,
+						@Nullable Boolean collateMatch) {
 					this.text = text;
 					this.highlighted = highlighted;
 					this.score = score;
@@ -121,11 +122,13 @@ public class Suggest {
 					return text;
 				}
 
+				@Nullable
 				public String getHighlighted() {
 					return highlighted;
 				}
 
-				public float getScore() {
+				@Nullable
+				public Double getScore() {
 					return score;
 				}
 

src/main/java/org/springframework/data/elasticsearch/core/suggest/response/TermSuggestion.java

@@ -15,6 +15,8 @@
  */
 package org.springframework.data.elasticsearch.core.suggest.response;
 
+import org.springframework.lang.Nullable;
+
 import java.util.List;
 
 /**
@@ -22,13 +24,14 @@ import java.util.List;
  */
 public class TermSuggestion extends Suggest.Suggestion<TermSuggestion.Entry> {
 
-	private final SortBy sort;
+	@Nullable private final SortBy sort;
 
-	public TermSuggestion(String name, int size, List<Entry> entries, SortBy sort) {
+	public TermSuggestion(String name, int size, List<Entry> entries, @Nullable SortBy sort) {
 		super(name, size, entries);
 		this.sort = sort;
 	}
 
+	@Nullable
 	public SortBy getSort() {
 		return sort;
 	}
@@ -43,7 +46,7 @@ public class TermSuggestion extends Suggest.Suggestion<TermSuggestion.Entry> {
 
 			private final int freq;
 
-			public Option(String text, String highlighted, float score, Boolean collateMatch, int freq) {
+			public Option(String text, @Nullable String highlighted, double score, @Nullable Boolean collateMatch, int freq) {
 				super(text, highlighted, score, collateMatch);
 				this.freq = freq;
 			}

src/main/java/org/springframework/data/elasticsearch/support/ScoreDoc.java

@@ -15,31 +15,35 @@
  */
 package org.springframework.data.elasticsearch.support;
 
+import org.springframework.lang.Nullable;
+
 /**
  * @author Peter-Josef Meisch
  * @since 4.3
  */
 public class ScoreDoc {
 
-	private final float score;
+	private final double score;
-	private final int doc;
+	@Nullable private final Integer doc;
-	private final int shardIndex;
+	@Nullable private final Integer shardIndex;
 
-	public ScoreDoc(float score, int doc, int shardIndex) {
+	public ScoreDoc(double score, @Nullable Integer doc, @Nullable Integer shardIndex) {
 		this.score = score;
 		this.doc = doc;
 		this.shardIndex = shardIndex;
 	}
 
-	public float getScore() {
+	public double getScore() {
 		return score;
 	}
 
-	public int getDoc() {
+	@Nullable
+	public Integer getDoc() {
 		return doc;
 	}
 
-	public int getShardIndex() {
+	@Nullable
+	public Integer getShardIndex() {
 		return shardIndex;
 	}
 }

src/test/java/org/springframework/data/elasticsearch/core/suggest/CompletionELCIntegrationTests.java

@@ -56,12 +56,10 @@ public class CompletionELCIntegrationTests extends CompletionIntegrationTests {
 		return NativeQuery.builder() //
 				.withSuggester(Suggester.of(s -> s //
 						.suggesters(suggestionName, FieldSuggester.of(fs -> fs //
+								.prefix(prefix)//
 								.completion(cs -> cs //
 										.field(fieldName) //
-										.prefix(prefix) //
 										.fuzzy(SuggestFuzziness.of(f -> f //
-												// NOTE we currently need to set all these values to their default as the client code does not
-												// have them nullable
 												.fuzziness("AUTO") //
 												.minLength(3) //
 												.prefixLength(1) //

src/test/java/org/springframework/data/elasticsearch/core/suggest/CompletionIntegrationTests.java

@@ -111,12 +111,12 @@ public abstract class CompletionIntegrationTests implements NewElasticsearchClie
 		indexQueries.add(new AnnotatedCompletionEntityBuilder("3").name("Mewes Kochheim3")
 				.suggest(new String[] { "Mewes Kochheim3" }, 2).buildIndex());
 		indexQueries.add(new AnnotatedCompletionEntityBuilder("4").name("Mewes Kochheim4")
-				.suggest(new String[] { "Mewes Kochheim4" }, Integer.MAX_VALUE).buildIndex());
+				.suggest(new String[] { "Mewes Kochheim4" }, 4444).buildIndex());
 
 		operations.bulkIndex(indexQueries, AnnotatedCompletionEntity.class);
 	}
 
-	@DisabledIf(value = "newElasticsearchClient", disabledReason = "todo #2139, ES issue 150")
+	// @DisabledIf(value = "newElasticsearchClient", disabledReason = "todo #2139, ES issue 150")
 	@Test
 	public void shouldFindSuggestionsForGivenCriteriaQueryUsingCompletionEntity() {
 
@@ -148,7 +148,7 @@ public abstract class CompletionIntegrationTests implements NewElasticsearchClie
 		operations.get("1", CompletionEntity.class);
 	}
 
-	@DisabledIf(value = "newElasticsearchClient", disabledReason = "todo #2139, ES issue 150")
+	// @DisabledIf(value = "newElasticsearchClient", disabledReason = "todo #2139, ES issue 150")
 	@Test
 	public void shouldFindSuggestionsForGivenCriteriaQueryUsingAnnotatedCompletionEntity() {
 
@@ -172,7 +172,7 @@ public abstract class CompletionIntegrationTests implements NewElasticsearchClie
 		assertThat(options.get(1).getText()).isIn("Marchand", "Mohsin");
 	}
 
-	@DisabledIf(value = "newElasticsearchClient", disabledReason = "todo #2139, ES 1issue 150")
+	// @DisabledIf(value = "newElasticsearchClient", disabledReason = "todo #2139, ES 1issue 150")
 	@Test
 	public void shouldFindSuggestionsWithWeightsForGivenCriteriaQueryUsingAnnotatedCompletionEntity() {
 
@@ -205,7 +205,7 @@ public abstract class CompletionIntegrationTests implements NewElasticsearchClie
 					assertThat(option.getScore()).isEqualTo(2);
 					break;
 				case "Mewes Kochheim4":
-					assertThat(option.getScore()).isEqualTo(Integer.MAX_VALUE);
+					assertThat(option.getScore()).isEqualTo(4444);
 					break;
 				default:
 					fail("Unexpected option");

src/test/java/org/springframework/data/elasticsearch/core/suggest/ReactiveSuggestELCIntegrationTests.java

@@ -54,9 +54,9 @@ public class ReactiveSuggestELCIntegrationTests extends ReactiveSuggestIntegrati
 		return NativeQuery.builder() //
 				.withSuggester(Suggester.of(s -> s //
 						.suggesters(suggestionName, FieldSuggester.of(fs -> fs //
+								.prefix(prefix) //
 								.completion(cs -> cs //
 										.field(fieldName) //
-										.prefix(prefix) //
 										.fuzzy(SuggestFuzziness.of(f -> f //
 												// NOTE we currently need to set all these values to their default as the client code does not
 												// have them nullable

src/test/java/org/springframework/data/elasticsearch/core/suggest/ReactiveSuggestIntegrationTests.java

@@ -67,7 +67,7 @@ public abstract class ReactiveSuggestIntegrationTests implements NewElasticsearc
 		operations.indexOps(IndexCoordinates.of(indexNameProvider.getPrefix() + "*")).delete().block();
 	}
 
-	@DisabledIf(value = "newElasticsearchClient", disabledReason = "todo #2139, ES issue 150")
+//	@DisabledIf(value = "newElasticsearchClient", disabledReason = "todo #2139, ES issue 150")
 	@Test // #1302
 	@DisplayName("should find suggestions for given prefix completion")
 	void shouldFindSuggestionsForGivenPrefixCompletion() {