honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix request cache key for search 

* Make sure indexBoost is serialized in a consistent order
* remove hasIndexBoost by using indexBoost size
* Make sure phrase suggester's collateParams is serialized in consistent
order
* Make StreamOutput writer to serialize maps in consistent order
This commit is contained in:
Xiang Chen 2016-08-19 23:24:17 +08:00 committed by Nik Everett
parent bd0b06440e
commit 22242ec881
5 changed files with 128 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
View File

@ -52,6 +52,7 @@ import java.nio.file.NotDirectoryException;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@ -413,6 +414,30 @@ public abstract class StreamOutput extends OutputStream {
writeGenericValue(map);
}
/**
* write map to stream with consistent order
* to make sure every map generated bytes order are same.
* This method is compatible with {@code StreamInput.readMap} and {@code StreamInput.readGenericValue}
* This method only will handle the map keys order, not maps contained within the map
*/
public void writeMapWithConsistentOrder(@Nullable Map<String, ? extends Object> map)
throws IOException {
if (map == null) {
writeByte((byte) -1);
return;
}
assert false == (map instanceof LinkedHashMap);
this.writeByte((byte) 10);
this.writeVInt(map.size());
Iterator<? extends Map.Entry<String, ?>> iterator =
map.entrySet().stream().sorted((a, b) -> a.getKey().compareTo(b.getKey())).iterator();
while (iterator.hasNext()) {
Map.Entry<String, ?> next = iterator.next();
this.writeString(next.getKey());
this.writeGenericValue(next.getValue());
}
}
/**
* Write a {@link Map} of {@code K}-type keys to {@code V}-type {@link List}s.
* <pre><code>
@ -553,6 +578,12 @@ public abstract class StreamOutput extends OutputStream {
WRITERS = Collections.unmodifiableMap(writers);
}
/**
* Notice: when serialization a map, the stream out map with the stream in map maybe have the
* different key-value orders, they will maybe have different stream order.
* If want to keep stream out map and stream in map have the same stream order when stream,
* can use {@code writeMapWithConsistentOrder}
*/
public void writeGenericValue(@Nullable Object value) throws IOException {
if (value == null) {
writeByte((byte) -1);

38
core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
View File

@ -20,13 +20,13 @@
package org.elasticsearch.search.builder;
import com.carrotsearch.hppc.ObjectFloatHashMap;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import org.elasticsearch.action.support.ToXContentToBytes;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
@ -63,6 +63,10 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import static org.elasticsearch.common.collect.Tuple.tuple;
/**
* A search source builder allowing to easily build search source. Simple
@ -188,11 +192,10 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
storedFieldsContext = in.readOptionalWriteable(StoredFieldsContext::new);
from = in.readVInt();
highlightBuilder = in.readOptionalWriteable(HighlightBuilder::new);
boolean hasIndexBoost = in.readBoolean();
if (hasIndexBoost) {
int size = in.readVInt();
indexBoost = new ObjectFloatHashMap<>(size);
for (int i = 0; i < size; i++) {
int indexBoostSize = in.readVInt();
if (indexBoostSize > 0) {
indexBoost = new ObjectFloatHashMap<>(indexBoostSize);
for (int i = 0; i < indexBoostSize; i++) {
indexBoost.put(in.readString(), in.readFloat());
}
}
@ -248,14 +251,10 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
out.writeOptionalWriteable(storedFieldsContext);
out.writeVInt(from);
out.writeOptionalWriteable(highlightBuilder);
boolean hasIndexBoost = indexBoost != null;
out.writeBoolean(hasIndexBoost);
if (hasIndexBoost) {
out.writeVInt(indexBoost.size());
for (ObjectCursor<String> key : indexBoost.keys()) {
out.writeString(key.value);
out.writeFloat(indexBoost.get(key.value));
}
int indexBoostSize = indexBoost == null ? 0 : indexBoost.size();
out.writeVInt(indexBoostSize);
if (indexBoostSize > 0) {
writeIndexBoost(out);
}
out.writeOptionalFloat(minScore);
out.writeOptionalNamedWriteable(postQueryBuilder);
@ -304,6 +303,17 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
out.writeOptionalWriteable(sliceBuilder);
}
private void writeIndexBoost(StreamOutput out) throws IOException {
List<Tuple<String, Float>> ibs = StreamSupport
.stream(indexBoost.spliterator(), false)
.map(i -> tuple(i.key, i.value)).sorted((o1, o2) -> o1.v1().compareTo(o2.v1()))
.collect(Collectors.toList());
for (Tuple<String, Float> ib : ibs) {
out.writeString(ib.v1());
out.writeFloat(ib.v2());
}
}
/**
* Sets the search query for this request.
*

2
core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
View File

@ -173,7 +173,7 @@ public class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSuggestionB
} else {
out.writeBoolean(false);
}
out.writeMap(collateParams);
out.writeMapWithConsistentOrder(collateParams);
out.writeOptionalBoolean(collatePrune);
out.writeVInt(this.generators.size());
for (Entry<String, List<CandidateGenerator>> entry : this.generators.entrySet()) {

51
core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
View File

@ -32,9 +32,14 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.TreeMap;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static org.hamcrest.Matchers.closeTo;
import static org.hamcrest.Matchers.endsWith;
@ -620,4 +625,50 @@ public class BytesStreamsTests extends ESTestCase {
out.writeBoolean(value);
}
}
public void testWriteMapWithConsistentOrder() throws IOException {
Map<String, String> map =
randomMap(new TreeMap<>(), randomIntBetween(2, 20),
() -> randomAsciiOfLength(5),
() -> randomAsciiOfLength(5));
Map<String, Object> reverseMap = new TreeMap<>(Collections.reverseOrder());
reverseMap.putAll(map);
List<String> mapKeys = map.entrySet().stream().map(Map.Entry::getKey).collect(Collectors.toList());
List<String> reverseMapKeys = reverseMap.entrySet().stream().map(Map.Entry::getKey).collect(Collectors.toList());
assertNotEquals(mapKeys, reverseMapKeys);
BytesStreamOutput output = new BytesStreamOutput();
BytesStreamOutput reverseMapOutput = new BytesStreamOutput();
output.writeMapWithConsistentOrder(map);
reverseMapOutput.writeMapWithConsistentOrder(reverseMap);
assertEquals(output.bytes(), reverseMapOutput.bytes());
}
public void testReadMapByUsingWriteMapWithConsistentOrder() throws IOException {
Map<String, String> streamOutMap =
randomMap(new HashMap<>(), randomIntBetween(2, 20),
() -> randomAsciiOfLength(5),
() -> randomAsciiOfLength(5));
BytesStreamOutput streamOut = new BytesStreamOutput();
streamOut.writeMapWithConsistentOrder(streamOutMap);
StreamInput in = StreamInput.wrap(BytesReference.toBytes(streamOut.bytes()));
Map<String, Object> streamInMap = in.readMap();
assertEquals(streamOutMap, streamInMap);
}
public void testWriteMapWithConsistentOrderWithLinkedHashMapShouldThrowAssertError() throws IOException {
BytesStreamOutput output = new BytesStreamOutput();
Map<String, Object> map = new LinkedHashMap<>();
Throwable e = expectThrows(AssertionError.class, () -> output.writeMapWithConsistentOrder(map));
assertEquals(AssertionError.class, e.getClass());
}
private static <K, V> Map<K, V> randomMap(Map<K, V> map, int size, Supplier<K> keyGenerator, Supplier<V> valueGenerator) {
IntStream.range(0, size).forEach(i -> map.put(keyGenerator.get(), valueGenerator.get()));
return map;
}
}

21
core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
View File

@ -666,4 +666,25 @@ public class SearchSourceBuilderTests extends ESTestCase {
String query = "{ \"query\": {} }";
assertParseSearchSource(builder, new BytesArray(query), ParseFieldMatcher.EMPTY);
}
public void testSearchRequestBuilderSerializationWithIndexBoost() throws Exception {
SearchSourceBuilder searchSourceBuilder = createSearchSourceBuilder();
createIndexBoost(searchSourceBuilder);
try (BytesStreamOutput output = new BytesStreamOutput()) {
searchSourceBuilder.writeTo(output);
try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
SearchSourceBuilder deserializedSearchSourceBuilder = new SearchSourceBuilder(in);
BytesStreamOutput deserializedOutput = new BytesStreamOutput();
deserializedSearchSourceBuilder.writeTo(deserializedOutput);
assertEquals(output.bytes(), deserializedOutput.bytes());
}
}
}
private void createIndexBoost(SearchSourceBuilder searchSourceBuilder) {
int indexBoostSize = randomIntBetween(1, 10);
for (int i = 0; i < indexBoostSize; i++) {
searchSourceBuilder.indexBoost(randomAsciiOfLengthBetween(5, 20), randomFloat() * 10);
}
}
}