Add a late-parsing SearchSoruce wrapper for index warmers

This commit is contained in:
Simon Willnauer 2015-09-23 21:29:11 +02:00
parent 2de450ca0b
commit 95a7b6aa83
8 changed files with 90 additions and 26 deletions

View File

@ -28,7 +28,6 @@ import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.warmer.IndexWarmersMetaData; import org.elasticsearch.search.warmer.IndexWarmersMetaData;
import java.io.IOException; import java.io.IOException;
@ -123,7 +122,7 @@ public class GetIndexResponse extends ActionResponse {
in.readString(), in.readString(),
in.readStringArray(), in.readStringArray(),
in.readOptionalBoolean(), in.readOptionalBoolean(),
in.readBoolean() ? SearchSourceBuilder.PROTOTYPE.readFrom(in) : null) in.readBoolean() ? new IndexWarmersMetaData.SearchSource(in) : null)
); );
} }
warmersMapBuilder.put(key, Collections.unmodifiableList(warmerEntryBuilder)); warmersMapBuilder.put(key, Collections.unmodifiableList(warmerEntryBuilder));

View File

@ -69,7 +69,7 @@ public class GetWarmersResponse extends ActionResponse {
for (int j = 0; j < valueSize; j++) { for (int j = 0; j < valueSize; j++) {
String name = in.readString(); String name = in.readString();
String[] types = in.readStringArray(); String[] types = in.readStringArray();
SearchSourceBuilder source = SearchSourceBuilder.PROTOTYPE.readFrom(in); IndexWarmersMetaData.SearchSource source = new IndexWarmersMetaData.SearchSource(in);
Boolean queryCache = null; Boolean queryCache = null;
queryCache = in.readOptionalBoolean(); queryCache = in.readOptionalBoolean();
warmerEntryBuilder.add(new IndexWarmersMetaData.Entry( warmerEntryBuilder.add(new IndexWarmersMetaData.Entry(

View File

@ -115,9 +115,9 @@ public class TransportPutWarmerAction extends TransportMasterNodeAction<PutWarme
MetaData metaData = currentState.metaData(); MetaData metaData = currentState.metaData();
String[] concreteIndices = indexNameExpressionResolver.concreteIndices(currentState, request.searchRequest().indicesOptions(), request.searchRequest().indices()); String[] concreteIndices = indexNameExpressionResolver.concreteIndices(currentState, request.searchRequest().indicesOptions(), request.searchRequest().indices());
SearchSourceBuilder source = null; IndexWarmersMetaData.SearchSource source = null;
if (request.searchRequest().source() != null) { if (request.searchRequest().source() != null) {
source = request.searchRequest().source(); source = new IndexWarmersMetaData.SearchSource(request.searchRequest().source());
} }
// now replace it on the metadata // now replace it on the metadata

View File

@ -21,6 +21,7 @@ package org.elasticsearch.common.xcontent;
import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.BytesReference;
import java.io.Closeable;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStream; import java.io.OutputStream;
@ -28,7 +29,7 @@ import java.io.OutputStream;
/** /**
* *
*/ */
public interface XContentGenerator { public interface XContentGenerator extends Closeable {
XContentType contentType(); XContentType contentType();

View File

@ -73,6 +73,7 @@ import org.elasticsearch.indices.IndicesWarmer;
import org.elasticsearch.indices.IndicesWarmer.TerminationHandle; import org.elasticsearch.indices.IndicesWarmer.TerminationHandle;
import org.elasticsearch.indices.IndicesWarmer.WarmerContext; import org.elasticsearch.indices.IndicesWarmer.WarmerContext;
import org.elasticsearch.indices.cache.request.IndicesRequestCache; import org.elasticsearch.indices.cache.request.IndicesRequestCache;
import org.elasticsearch.indices.query.IndicesQueriesRegistry;
import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.node.settings.NodeSettingsService;
import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ExecutableScript;
import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptContext;
@ -1157,8 +1158,9 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> {
SearchContext context = null; SearchContext context = null;
try { try {
long now = System.nanoTime(); long now = System.nanoTime();
final IndexService indexService = indicesService.indexServiceSafe(indexShard.shardId().index().name());
ShardSearchRequest request = new ShardSearchLocalRequest(indexShard.shardId(), indexMetaData.numberOfShards(), ShardSearchRequest request = new ShardSearchLocalRequest(indexShard.shardId(), indexMetaData.numberOfShards(),
SearchType.QUERY_THEN_FETCH, entry.source(), entry.types(), entry.requestCache()); SearchType.QUERY_THEN_FETCH, entry.source().build(new QueryParseContext(indexService.queryParserService().indicesQueriesRegistry())), entry.types(), entry.requestCache());
context = createContext(request, warmerContext.searcher()); context = createContext(request, warmerContext.searcher());
// if we use sort, we need to do query to sort on // if we use sort, we need to do query to sort on
// it and load relevant field data // it and load relevant field data

View File

@ -19,19 +19,22 @@
package org.elasticsearch.search.warmer; package org.elasticsearch.search.warmer;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.cluster.AbstractDiffable;
import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings; import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.util.ByteArray;
import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.*;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder;
import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
@ -66,10 +69,10 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
public static class Entry { public static class Entry {
private final String name; private final String name;
private final String[] types; private final String[] types;
private final SearchSourceBuilder source; private final SearchSource source;
private final Boolean requestCache; private final Boolean requestCache;
public Entry(String name, String[] types, Boolean requestCache, SearchSourceBuilder source) { public Entry(String name, String[] types, Boolean requestCache, SearchSource source) {
this.name = name; this.name = name;
this.types = types == null ? Strings.EMPTY_ARRAY : types; this.types = types == null ? Strings.EMPTY_ARRAY : types;
this.source = source; this.source = source;
@ -85,7 +88,7 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
} }
@Nullable @Nullable
public SearchSourceBuilder source() { public SearchSource source() {
return this.source; return this.source;
} }
@ -140,9 +143,9 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
for (int i = 0; i < entries.length; i++) { for (int i = 0; i < entries.length; i++) {
String name = in.readString(); String name = in.readString();
String[] types = in.readStringArray(); String[] types = in.readStringArray();
SearchSourceBuilder source = null; SearchSource source = null;
if (in.readBoolean()) { if (in.readBoolean()) {
source = SearchSourceBuilder.PROTOTYPE.readFrom(in); source = new SearchSource(in);
} }
Boolean queryCache; Boolean queryCache;
queryCache = in.readOptionalBoolean(); queryCache = in.readOptionalBoolean();
@ -193,7 +196,7 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
} else if (token == XContentParser.Token.START_OBJECT) { } else if (token == XContentParser.Token.START_OBJECT) {
String name = currentFieldName; String name = currentFieldName;
List<String> types = new ArrayList<>(2); List<String> types = new ArrayList<>(2);
SearchSourceBuilder source = null; SearchSource source = null;
Boolean queryCache = null; Boolean queryCache = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) { if (token == XContentParser.Token.FIELD_NAME) {
@ -206,12 +209,16 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
} }
} else if (token == XContentParser.Token.START_OBJECT) { } else if (token == XContentParser.Token.START_OBJECT) {
if ("source".equals(currentFieldName)) { if ("source".equals(currentFieldName)) {
source = SearchSourceBuilder.PROTOTYPE.fromXContent(parser, null); // NOCOMMIT need context from somewhere ByteArrayOutputStream out = new ByteArrayOutputStream();
try (XContentGenerator generator = parser.contentType().xContent().createGenerator(out)) {
generator.copyCurrentStructure(parser);
}
source = new SearchSource(new BytesArray(out.toByteArray()));
}
} else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
if ("source".equals(currentFieldName)) {
source = new SearchSource(new BytesArray(parser.binaryValue()));
} }
// } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
// if ("source".equals(currentFieldName)) {
// source = new BytesArray(parser.binaryValue());
// } NORELEASE do we need this?
} else if (token.isValue()) { } else if (token.isValue()) {
if ("requestCache".equals(currentFieldName) || "request_cache".equals(currentFieldName)) { if ("requestCache".equals(currentFieldName) || "request_cache".equals(currentFieldName)) {
queryCache = parser.booleanValue(); queryCache = parser.booleanValue();
@ -265,4 +272,59 @@ public class IndexWarmersMetaData extends AbstractDiffable<IndexMetaData.Custom>
} }
return new IndexWarmersMetaData(entries.toArray(new Entry[entries.size()])); return new IndexWarmersMetaData(entries.toArray(new Entry[entries.size()]));
} }
public static class SearchSource implements ToXContent, Writeable<SearchSource> {
private final BytesReference binary;
private SearchSourceBuilder cached;
public SearchSource(BytesReference bytesArray) {
this.binary = bytesArray;
}
public SearchSource(StreamInput input) throws IOException {
this(input.readBytesReference());
}
public SearchSource(SearchSourceBuilder source) {
try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
source.toXContent(builder, ToXContent.EMPTY_PARAMS);
binary = builder.bytes();
} catch (IOException ex) {
throw new ElasticsearchException("failed to generate XContent", ex);
}
}
public SearchSourceBuilder build(QueryParseContext ctx) throws IOException {
if (cached == null) {
try (XContentParser parser = XContentFactory.xContent(binary).createParser(binary)) {
ctx.reset(parser);
cached = SearchSourceBuilder.PROTOTYPE.fromXContent(parser, ctx);
}
}
return cached;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
if (binary == null) {
cached.toXContent(builder, params);
} else {
try (XContentParser parser = XContentFactory.xContent(binary).createParser(binary)) {
builder.copyCurrentStructure(parser);
}
}
return builder;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeBytesReference(binary);
}
@Override
public SearchSource readFrom(StreamInput in) throws IOException {
return new SearchSource(in);
}
}
} }

View File

@ -533,7 +533,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase {
randomName("warm"), randomName("warm"),
new String[]{randomName("type")}, new String[]{randomName("type")},
randomBoolean(), randomBoolean(),
new SearchSourceBuilder()) // NOCOMMIT this used to be new BytesArray(randomAsciiOfLength(1000)) whiat should it be now? new IndexWarmersMetaData.SearchSource(new BytesArray(randomAsciiOfLength(1000))))
); );
} else { } else {
return new IndexWarmersMetaData(); return new IndexWarmersMetaData();

View File

@ -677,7 +677,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
@Test @Test
public void testDeleteWarmer() throws Exception { public void testDeleteWarmer() throws Exception {
SearchSourceBuilder source = new SearchSourceBuilder(); SearchSourceBuilder source = new SearchSourceBuilder();
IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry("test1", new String[] { "typ1" }, false, source); IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry("test1", new String[] { "typ1" }, false, new IndexWarmersMetaData.SearchSource(source));
assertAcked(prepareCreate("foobar").addCustom(new IndexWarmersMetaData(entry))); assertAcked(prepareCreate("foobar").addCustom(new IndexWarmersMetaData(entry)));
ensureYellow(); ensureYellow();
@ -692,7 +692,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
verify(client().admin().indices().prepareDeleteWarmer().setIndices("_all").setNames("test1"), true); verify(client().admin().indices().prepareDeleteWarmer().setIndices("_all").setNames("test1"), true);
SearchSourceBuilder source = new SearchSourceBuilder(); SearchSourceBuilder source = new SearchSourceBuilder();
IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry("test1", new String[] { "type1" }, false, source); IndexWarmersMetaData.Entry entry = new IndexWarmersMetaData.Entry("test1", new String[] { "type1" }, false, new IndexWarmersMetaData.SearchSource(source));
assertAcked(prepareCreate("foo").addCustom(new IndexWarmersMetaData(entry))); assertAcked(prepareCreate("foo").addCustom(new IndexWarmersMetaData(entry)));
assertAcked(prepareCreate("foobar").addCustom(new IndexWarmersMetaData(entry))); assertAcked(prepareCreate("foobar").addCustom(new IndexWarmersMetaData(entry)));
assertAcked(prepareCreate("bar").addCustom(new IndexWarmersMetaData(entry))); assertAcked(prepareCreate("bar").addCustom(new IndexWarmersMetaData(entry)));