Cwikius-Spring/spring-data-elasticsearch
1
0
Fork 0
mirror of https://github.com/spring-projects/spring-data-elasticsearch.git synced 2025-06-20 19:12:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

DATAES-510 - Polishing.

Browse Source 
Wrap Scroll execution with usingWhen and run cleanup through usingWhen callback to clean up scrolls state on success/on error/on cancellation.

Extract isEmpty(SearchHits) check into own method. Improve synchronization in ScrollState to prevent concurrent modification exceptions during read.

Original Pull Request: #231
This commit is contained in:
Mark Paluch 2018-12-10 12:03:38 +01:00 committed by Christoph Strobl
parent da9de6bc49
commit b50e60fdd1
1 changed files with 45 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
src/main/java/org/springframework/data/elasticsearch/client/reactive/DefaultReactiveElasticsearchClient.java
View File

@ -78,6 +78,7 @@ import org.elasticsearch.rest.BytesRestResponse;
import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.Scroll; import org.elasticsearch.search.Scroll;
import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.reactivestreams.Publisher; import org.reactivestreams.Publisher;
import org.springframework.data.elasticsearch.ElasticsearchException; import org.springframework.data.elasticsearch.ElasticsearchException;
import org.springframework.data.elasticsearch.client.ClientConfiguration; import org.springframework.data.elasticsearch.client.ClientConfiguration;
@ -92,6 +93,7 @@ import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType; import org.springframework.http.MediaType;
import org.springframework.http.client.reactive.ReactorClientHttpConnector; import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.lang.Nullable;
import org.springframework.util.Assert; import org.springframework.util.Assert;
import org.springframework.util.ObjectUtils; import org.springframework.util.ObjectUtils;
import org.springframework.util.ReflectionUtils; import org.springframework.util.ReflectionUtils;
@ -339,41 +341,55 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
} }
throw new IllegalArgumentException( throw new IllegalArgumentException(
String.format("Cannot handle '%s'. Please make sure to use a 'SearchRequest' or 'SearchScrollRequest'.")); String.format("Cannot handle '%s'. Please make sure to use a 'SearchRequest' or 'SearchScrollRequest'.", it));
}); });
ScrollState state = new ScrollState(); return Flux.usingWhen(Mono.fromSupplier(ScrollState::new),
Flux<SearchHit> searchHits = inbound.doOnNext(searchResponse -> { scrollState -> {
state.updateScrollId(searchResponse.getScrollId());
}).<SearchResponse> handle((searchResponse, sink) -> {
if (searchResponse.getHits() != null && searchResponse.getHits().getHits() != null Flux<SearchHit> searchHits = inbound.<SearchResponse> handle((searchResponse, sink) -> {
&& searchResponse.getHits().getHits().length == 0) {
inbound.onComplete(); scrollState.updateScrollId(searchResponse.getScrollId());
outbound.onComplete(); if (isEmpty(searchResponse.getHits())) {
} else { inbound.onComplete();
outbound.onComplete();
sink.next(searchResponse); } else {
SearchScrollRequest searchScrollRequest = new SearchScrollRequest(state.getScrollId()).scroll(scrollTimeout); sink.next(searchResponse);
request.next(searchScrollRequest);
}
}).map(SearchResponse::getHits) // SearchScrollRequest searchScrollRequest = new SearchScrollRequest(scrollState.getScrollId())
.flatMap(Flux::fromIterable) // .scroll(scrollTimeout);
.doOnComplete(() -> { request.next(searchScrollRequest);
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); }).map(SearchResponse::getHits) //
clearScrollRequest.scrollIds(state.getScrollIds()); .flatMap(Flux::fromIterable);
// just send the request, resources get cleaned up anyways after scrollTimeout has been reached. return searchHits.doOnSubscribe(ignore -> exchange.subscribe(inbound));
sendRequest(clearScrollRequest, RequestCreator.clearScroll(), ClearScrollResponse.class, headers).subscribe();
});
return searchHits.doOnSubscribe(ignore -> exchange.subscribe(inbound)); }, state -> cleanupScroll(headers, state), //
state -> cleanupScroll(headers, state), //
state -> cleanupScroll(headers, state)); //
}
private static boolean isEmpty(@Nullable SearchHits hits) {
return hits != null && hits.getHits() != null && hits.getHits().length == 0;
}
private Publisher<?> cleanupScroll(HttpHeaders headers, ScrollState state) {
if (state.getScrollIds().isEmpty()) {
return Mono.empty();
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.scrollIds(state.getScrollIds());
// just send the request, resources get cleaned up anyways after scrollTimeout has been reached.
return sendRequest(clearScrollRequest, RequestCreator.clearScroll(), ClearScrollResponse.class, headers);
} }
/* /*
@ -645,17 +661,20 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
*/ */
private static class ScrollState { private static class ScrollState {
private Object lock = new Object(); private final Object lock = new Object();
private final List<String> pastIds = new ArrayList<>(1);
private String scrollId; private String scrollId;
private List<String> pastIds = new ArrayList<>(1);
String getScrollId() { String getScrollId() {
return scrollId; return scrollId;
} }
List<String> getScrollIds() { List<String> getScrollIds() {
return Collections.unmodifiableList(pastIds);
synchronized (lock) {
return Collections.unmodifiableList(new ArrayList<>(pastIds));
}
} }
void updateScrollId(String scrollId) { void updateScrollId(String scrollId) {
@ -669,6 +688,5 @@ public class DefaultReactiveElasticsearchClient implements ReactiveElasticsearch
} }
} }
} }
} }
} }