honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add a late-parsing SearchSoruce wrapper for index warmers

This commit is contained in:
Simon Willnauer 2015-09-23 21:29:11 +02:00
parent 2de450ca0b
commit 95a7b6aa83
8 changed files with 90 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
core/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexResponse.java
View File

@ -28,7 +28,6 @@ import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.warmer.IndexWarmersMetaData;
import java.io.IOException;
@ -123,7 +122,7 @@ public class GetIndexResponse extends ActionResponse {
in.readString(),
in.readStringArray(),
in.readOptionalBoolean(),
in.readBoolean() ? SearchSourceBuilder.PROTOTYPE.readFrom(in) : null)
in.readBoolean() ? new IndexWarmersMetaData.SearchSource(in) : null)
);
}
warmersMapBuilder.put(key, Collections.unmodifiableList(warmerEntryBuilder));

2
core/src/main/java/org/elasticsearch/action/admin/indices/warmer/get/GetWarmersResponse.java
View File

@ -69,7 +69,7 @@ public class GetWarmersResponse extends ActionResponse {
for (int j = 0; j < valueSize; j++) {
String name = in.readString();
String[] types = in.readStringArray();
SearchSourceBuilder source = SearchSourceBuilder.PROTOTYPE.readFrom(in);
IndexWarmersMetaData.SearchSource source = new IndexWarmersMetaData.SearchSource(in);
Boolean queryCache = null;
queryCache = in.readOptionalBoolean();
warmerEntryBuilder.add(new IndexWarmersMetaData.Entry(

4
core/src/main/java/org/elasticsearch/action/admin/indices/warmer/put/TransportPutWarmerAction.java
View File

@ -115,9 +115,9 @@ public class TransportPutWarmerAction extends TransportMasterNodeAction<PutWarme
MetaData metaData = currentState.metaData();
String[] concreteIndices = indexNameExpressionResolver.concreteIndices(currentState, request.searchRequest().indicesOptions(), request.searchRequest().indices());
SearchSourceBuilder source = null;
IndexWarmersMetaData.SearchSource source = null;
if (request.searchRequest().source() != null) {
source = request.searchRequest().source();
source = new IndexWarmersMetaData.SearchSource(request.searchRequest().source());
}
// now replace it on the metadata

3
core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.common.xcontent;
import org.elasticsearch.common.bytes.BytesReference;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@ -28,7 +29,7 @@ import java.io.OutputStream;
/**
*
*/
public interface XContentGenerator {
public interface XContentGenerator extends Closeable {
XContentType contentType();

4
core/src/main/java/org/elasticsearch/search/SearchService.java
View File

@ -73,6 +73,7 @@ import org.elasticsearch.indices.IndicesWarmer;
import org.elasticsearch.indices.IndicesWarmer.TerminationHandle;
import org.elasticsearch.indices.IndicesWarmer.WarmerContext;
import org.elasticsearch.indices.cache.request.IndicesRequestCache;
import org.elasticsearch.indices.query.IndicesQueriesRegistry;
import org.elasticsearch.node.settings.NodeSettingsService;
import org.elasticsearch.script.ExecutableScript;
import org.elasticsearch.script.ScriptContext;
@ -1157,8 +1158,9 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
SearchContext context = null;
try {
long now = System.nanoTime();
final IndexService indexService = indicesService.indexServiceSafe(indexShard.shardId().index().name());
ShardSearchRequest request = new ShardSearchLocalRequest(indexShard.shardId(), indexMetaData.numberOfShards(),
SearchType.QUERY_THEN_FETCH, entry.source(), entry.types(), entry.requestCache());
SearchType.QUERY_THEN_FETCH, entry.source().build(new QueryParseContext(indexService.queryParserService().indicesQueriesRegistry())), entry.types(), entry.requestCache());
context = createContext(request, warmerContext.searcher());
// if we use sort, we need to do query to sort on
// it and load relevant field data

94
core/src/main/java/org/elasticsearch/search/warmer/IndexWarmersMetaData.java
View File

@ -19,19 +19,22 @@
package org.elasticsearch.search.warmer;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cluster.AbstractDiffable;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.util.ByteArray;
import org.elasticsearch.common.xcontent.*;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@ -66,10 +69,10 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
public static class Entry {
private final String name;
private final String[] types;
private final SearchSourceBuilder source;
private final SearchSource source;
private final Boolean requestCache;
public Entry(String name, String[] types, Boolean requestCache, SearchSourceBuilder source) {
public Entry(String name, String[] types, Boolean requestCache, SearchSource source) {
this.name = name;
this.types = types == null ? Strings.EMPTY_ARRAY : types;
this.source = source;
@ -85,7 +88,7 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
}
@Nullable
public SearchSourceBuilder source() {
public SearchSource source() {
return this.source;
}
@ -140,9 +143,9 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
for (int i = 0; i < entries.length; i++) {
String name = in.readString();
String[] types = in.readStringArray();
SearchSourceBuilder source = null;
SearchSource source = null;
if (in.readBoolean()) {
source = SearchSourceBuilder.PROTOTYPE.readFrom(in);
source = new SearchSource(in);
}
Boolean queryCache;
queryCache = in.readOptionalBoolean();
@ -193,7 +196,7 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
} else if (token == XContentParser.Token.START_OBJECT) {
String name = currentFieldName;
List<String> types = new ArrayList<>(2);
SearchSourceBuilder source = null;
SearchSource source = null;
Boolean queryCache = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
@ -206,12 +209,16 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
}
} else if (token == XContentParser.Token.START_OBJECT) {
if ("source".equals(currentFieldName)) {
source = SearchSourceBuilder.PROTOTYPE.fromXContent(parser, null); // NOCOMMIT need context from somewhere
ByteArrayOutputStream out = new ByteArrayOutputStream();
try (XContentGenerator generator = parser.contentType().xContent().createGenerator(out)) {
generator.copyCurrentStructure(parser);
}
source = new SearchSource(new BytesArray(out.toByteArray()));
}
} else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
if ("source".equals(currentFieldName)) {
source = new SearchSource(new BytesArray(parser.binaryValue()));
}
// } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
// if ("source".equals(currentFieldName)) {
// source = new BytesArray(parser.binaryValue());
// } NORELEASE do we need this?
} else if (token.isValue()) {
if ("requestCache".equals(currentFieldName) || "request_cache".equals(currentFieldName)) {
queryCache = parser.booleanValue();
@ -265,4 +272,59 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
}
return new IndexWarmersMetaData(entries.toArray(new Entry[entries.size()]));
}
public static class SearchSource implements ToXContent, Writeable<SearchSource> {
private final BytesReference binary;
private SearchSourceBuilder cached;
public SearchSource(BytesReference bytesArray) {
this.binary = bytesArray;
}
public SearchSource(StreamInput input) throws IOException {
this(input.readBytesReference());
}
public SearchSource(SearchSourceBuilder source) {
try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
source.toXContent(builder, ToXContent.EMPTY_PARAMS);
binary = builder.bytes();
} catch (IOException ex) {
throw new ElasticsearchException("failed to generate XContent", ex);
}
}
public SearchSourceBuilder build(QueryParseContext ctx) throws IOException {
if (cached == null) {
try (XContentParser parser = XContentFactory.xContent(binary).createParser(binary)) {
ctx.reset(parser);
cached = SearchSourceBuilder.PROTOTYPE.fromXContent(parser, ctx);
}
}
return cached;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
if (binary == null) {
cached.toXContent(builder, params);
} else {
try (XContentParser parser = XContentFactory.xContent(binary).createParser(binary)) {
builder.copyCurrentStructure(parser);
}
}
return builder;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeBytesReference(binary);
}
@Override
public SearchSource readFrom(StreamInput in) throws IOException {
return new SearchSource(in);
}
}
}

2
core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
View File

@ -533,7 +533,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase {
randomName("warm"),
new String[]{randomName("type")},
randomBoolean(),
new SearchSourceBuilder()) // NOCOMMIT this used to be new BytesArray(randomAsciiOfLength(1000)) whiat should it be now?
new IndexWarmersMetaData.SearchSource(new BytesArray(randomAsciiOfLength(1000))))
);
} else {
return new IndexWarmersMetaData();

4
core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
View File

@ -677,7 +677,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
@Test
public void testDeleteWarmer() throws Exception {
SearchSourceBuilder source = new SearchSourceBuilder();
IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry("test1", new String[] { "typ1" }, false, source);
IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry("test1", new String[] { "typ1" }, false, new IndexWarmersMetaData.SearchSource(source));
assertAcked(prepareCreate("foobar").addCustom(new IndexWarmersMetaData(entry)));
ensureYellow();
@ -692,7 +692,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
verify(client().admin().indices().prepareDeleteWarmer().setIndices("_all").setNames("test1"), true);
SearchSourceBuilder source = new SearchSourceBuilder();
IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry("test1", new String[] { "type1" }, false, source);
IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry("test1", new String[] { "type1" }, false, new IndexWarmersMetaData.SearchSource(source));
assertAcked(prepareCreate("foo").addCustom(new IndexWarmersMetaData(entry)));
assertAcked(prepareCreate("foobar").addCustom(new IndexWarmersMetaData(entry)));
assertAcked(prepareCreate("bar").addCustom(new IndexWarmersMetaData(entry)));