Search: Support partial fields that can returns partial view of the _source, closes #1570.

This commit is contained in:
Shay Banon 2011-12-26 16:49:55 +02:00
parent 37398c2000
commit cc3f44473f
14 changed files with 574 additions and 213 deletions

View File

@ -27,6 +27,7 @@ import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.action.support.BaseRequestBuilder;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.FilterBuilder;
@ -392,6 +393,32 @@ public class SearchRequestBuilder extends BaseRequestBuilder<SearchRequest, Sear
return this;
}
/**
* Adds a partial field based on _source, with an "include" and/or "exclude" set which can include simple wildcard
* elements.
*
* @param name The name of the field
* @param include An optional include (optionally wildcarded) pattern from _source
* @param exclude An optional exclude (optionally wildcarded) pattern from _source
*/
public SearchRequestBuilder addPartialField(String name, @Nullable String include, @Nullable String exclude) {
sourceBuilder().partialField(name, include, exclude);
return this;
}
/**
* Adds a partial field based on _source, with an "includes" and/or "excludes set which can include simple wildcard
* elements.
*
* @param name The name of the field
* @param includes An optional list of includes (optionally wildcarded) patterns from _source
* @param excludes An optional list of excludes (optionally wildcarded) patterns from _source
*/
public SearchRequestBuilder addPartialField(String name, @Nullable String[] includes, @Nullable String[] excludes) {
sourceBuilder().partialField(name, includes, excludes);
return this;
}
/**
* Adds a script based field to load and return. The field does not have to be stored,
* but its recommended to use non analyzed or numeric fields.

View File

@ -146,7 +146,9 @@ public class XContentMapValues {
if (includes.length > 0) {
boolean atLeastOnOneIncludeMatched = false;
for (String include : includes) {
if (Regex.simpleMatch(include, path)) {
// check for prefix as well, something like: obj1.arr1.*
// note, this does not work well with middle matches, like obj1.*.obj3
if (include.startsWith(path) || Regex.simpleMatch(include, path)) {
atLeastOnOneIncludeMatched = true;
break;
}

View File

@ -26,7 +26,6 @@ import java.util.List;
/**
* A single field name and values part of a {@link SearchHit}.
*
*
* @see SearchHit
*/
public interface SearchHitField extends Streamable, Iterable<Object> {
@ -44,12 +43,12 @@ public interface SearchHitField extends Streamable, Iterable<Object> {
/**
* The first value of the hit.
*/
Object value();
<V> V value();
/**
* The first value of the hit.
*/
Object getValue();
<V> V getValue();
/**
* The field values.

View File

@ -30,6 +30,7 @@ import org.elasticsearch.search.facet.FacetModule;
import org.elasticsearch.search.fetch.FetchPhase;
import org.elasticsearch.search.fetch.explain.ExplainFetchSubPhase;
import org.elasticsearch.search.fetch.matchedfilters.MatchedFiltersFetchSubPhase;
import org.elasticsearch.search.fetch.partial.PartialFieldsFetchSubPhase;
import org.elasticsearch.search.fetch.script.ScriptFieldsFetchSubPhase;
import org.elasticsearch.search.fetch.version.VersionFetchSubPhase;
import org.elasticsearch.search.highlight.HighlightPhase;
@ -55,6 +56,7 @@ public class SearchModule extends AbstractModule implements SpawnModules {
bind(FetchPhase.class).asEagerSingleton();
bind(ExplainFetchSubPhase.class).asEagerSingleton();
bind(ScriptFieldsFetchSubPhase.class).asEagerSingleton();
bind(PartialFieldsFetchSubPhase.class).asEagerSingleton();
bind(VersionFetchSubPhase.class).asEagerSingleton();
bind(MatchedFiltersFetchSubPhase.class).asEagerSingleton();
bind(HighlightPhase.class).asEagerSingleton();

View File

@ -25,6 +25,7 @@ import gnu.trove.iterator.TObjectFloatIterator;
import gnu.trove.map.hash.TObjectFloatHashMap;
import org.elasticsearch.ElasticSearchGenerationException;
import org.elasticsearch.client.Requests;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Unicode;
import org.elasticsearch.common.io.BytesStream;
import org.elasticsearch.common.xcontent.ToXContent;
@ -48,7 +49,6 @@ import java.util.Map;
* A search source builder allowing to easily build search source. Simple construction
* using {@link org.elasticsearch.search.builder.SearchSourceBuilder#searchSource()}.
*
*
* @see org.elasticsearch.action.search.SearchRequest#source(SearchSourceBuilder)
*/
public class SearchSourceBuilder implements ToXContent {
@ -94,8 +94,8 @@ public class SearchSourceBuilder implements ToXContent {
private Float minScore;
private List<String> fieldNames;
private List<ScriptField> scriptFields;
private List<PartialField> partialFields;
private List<AbstractFacetBuilder> facets;
@ -463,6 +463,38 @@ public class SearchSourceBuilder implements ToXContent {
return this;
}
/**
* Adds a partial field based on _source, with an "include" and/or "exclude" set which can include simple wildcard
* elements.
*
* @param name The name of the field
* @param include An optional include (optionally wildcarded) pattern from _source
* @param exclude An optional exclude (optionally wildcarded) pattern from _source
*/
public SearchSourceBuilder partialField(String name, @Nullable String include, @Nullable String exclude) {
if (partialFields == null) {
partialFields = Lists.newArrayList();
}
partialFields.add(new PartialField(name, include, exclude));
return this;
}
/**
* Adds a partial field based on _source, with an "includes" and/or "excludes set which can include simple wildcard
* elements.
*
* @param name The name of the field
* @param includes An optional list of includes (optionally wildcarded) patterns from _source
* @param excludes An optional list of excludes (optionally wildcarded) patterns from _source
*/
public SearchSourceBuilder partialField(String name, @Nullable String[] includes, @Nullable String[] excludes) {
if (partialFields == null) {
partialFields = Lists.newArrayList();
}
partialFields.add(new PartialField(name, includes, excludes));
return this;
}
/**
* Sets the boost a specific index will receive when the query is executeed against it.
*
@ -582,6 +614,29 @@ public class SearchSourceBuilder implements ToXContent {
}
}
if (partialFields != null) {
builder.startObject("partial_fields");
for (PartialField partialField : partialFields) {
builder.startObject(partialField.name());
if (partialField.includes() != null) {
if (partialField.includes().length == 1) {
builder.field("include", partialField.includes()[0]);
} else {
builder.field("include", partialField.includes());
}
}
if (partialField.excludes() != null) {
if (partialField.excludes().length == 1) {
builder.field("exclude", partialField.excludes()[0]);
} else {
builder.field("exclude", partialField.excludes());
}
}
builder.endObject();
}
builder.endObject();
}
if (scriptFields != null) {
builder.startObject("script_fields");
for (ScriptField scriptField : scriptFields) {
@ -683,4 +738,34 @@ public class SearchSourceBuilder implements ToXContent {
return params;
}
}
private static class PartialField {
private final String name;
private final String[] includes;
private final String[] excludes;
private PartialField(String name, String[] includes, String[] excludes) {
this.name = name;
this.includes = includes;
this.excludes = excludes;
}
private PartialField(String name, String include, String exclude) {
this.name = name;
this.includes = include == null ? null : new String[]{include};
this.excludes = exclude == null ? null : new String[]{exclude};
}
public String name() {
return name;
}
public String[] includes() {
return includes;
}
public String[] excludes() {
return excludes;
}
}
}

View File

@ -44,6 +44,7 @@ import org.elasticsearch.search.SearchParseElement;
import org.elasticsearch.search.SearchPhase;
import org.elasticsearch.search.fetch.explain.ExplainFetchSubPhase;
import org.elasticsearch.search.fetch.matchedfilters.MatchedFiltersFetchSubPhase;
import org.elasticsearch.search.fetch.partial.PartialFieldsFetchSubPhase;
import org.elasticsearch.search.fetch.script.ScriptFieldsFetchSubPhase;
import org.elasticsearch.search.fetch.version.VersionFetchSubPhase;
import org.elasticsearch.search.highlight.HighlightPhase;
@ -66,9 +67,9 @@ public class FetchPhase implements SearchPhase {
private final FetchSubPhase[] fetchSubPhases;
@Inject
public FetchPhase(HighlightPhase highlightPhase, ScriptFieldsFetchSubPhase scriptFieldsPhase,
public FetchPhase(HighlightPhase highlightPhase, ScriptFieldsFetchSubPhase scriptFieldsPhase, PartialFieldsFetchSubPhase partialFieldsPhase,
MatchedFiltersFetchSubPhase matchFiltersPhase, ExplainFetchSubPhase explainPhase, VersionFetchSubPhase versionPhase) {
this.fetchSubPhases = new FetchSubPhase[]{scriptFieldsPhase, matchFiltersPhase, explainPhase, highlightPhase, versionPhase};
this.fetchSubPhases = new FetchSubPhase[]{scriptFieldsPhase, partialFieldsPhase, matchFiltersPhase, explainPhase, highlightPhase, versionPhase};
}
@Override
@ -89,13 +90,19 @@ public class FetchPhase implements SearchPhase {
ResetFieldSelector fieldSelector;
List<String> extractFieldNames = null;
boolean sourceRequested = false;
if (context.hasScriptFields() && !context.hasFieldNames()) {
if (!context.hasFieldNames()) {
if (context.hasPartialFields()) {
// partial fields need the source, so fetch it, but don't return it
fieldSelector = new UidAndSourceFieldSelector();
sourceRequested = false;
} else if (context.hasScriptFields()) {
// we ask for script fields, and no field names, don't load the source
fieldSelector = UidFieldSelector.INSTANCE;
sourceRequested = false;
} else if (!context.hasFieldNames()) {
} else {
fieldSelector = new UidAndSourceFieldSelector();
sourceRequested = true;
}
} else if (context.fieldNames().isEmpty()) {
fieldSelector = UidFieldSelector.INSTANCE;
sourceRequested = false;

View File

@ -0,0 +1,67 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.fetch.partial;
import com.google.common.collect.Lists;
import java.util.List;
/**
*/
public class PartialFieldsContext {
public static class PartialField {
private final String name;
private final String[] includes;
private final String[] excludes;
public PartialField(String name, String[] includes, String[] excludes) {
this.name = name;
this.includes = includes;
this.excludes = excludes;
}
public String name() {
return this.name;
}
public String[] includes() {
return this.includes;
}
public String[] excludes() {
return this.excludes;
}
}
private final List<PartialField> fields = Lists.newArrayList();
public PartialFieldsContext() {
}
public void add(PartialField field) {
fields.add(field);
}
public List<PartialField> fields() {
return this.fields;
}
}

View File

@ -0,0 +1,84 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.fetch.partial;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.search.SearchHitField;
import org.elasticsearch.search.SearchParseElement;
import org.elasticsearch.search.fetch.FetchSubPhase;
import org.elasticsearch.search.internal.InternalSearchHit;
import org.elasticsearch.search.internal.InternalSearchHitField;
import org.elasticsearch.search.internal.SearchContext;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
/**
*/
public class PartialFieldsFetchSubPhase implements FetchSubPhase {
@Inject
public PartialFieldsFetchSubPhase() {
}
@Override
public Map<String, ? extends SearchParseElement> parseElements() {
ImmutableMap.Builder<String, SearchParseElement> parseElements = ImmutableMap.builder();
parseElements.put("partial_fields", new PartialFieldsParseElement())
.put("partialFields", new PartialFieldsParseElement());
return parseElements.build();
}
@Override
public boolean hitsExecutionNeeded(SearchContext context) {
return false;
}
@Override
public void hitsExecute(SearchContext context, InternalSearchHit[] hits) throws ElasticSearchException {
}
@Override
public boolean hitExecutionNeeded(SearchContext context) {
return context.hasPartialFields();
}
@Override
public void hitExecute(SearchContext context, HitContext hitContext) throws ElasticSearchException {
for (PartialFieldsContext.PartialField field : context.partialFields().fields()) {
Object value = context.lookup().source().filter(field.includes(), field.excludes());
if (hitContext.hit().fieldsOrNull() == null) {
hitContext.hit().fields(new HashMap<String, SearchHitField>(2));
}
SearchHitField hitField = hitContext.hit().fields().get(field.name());
if (hitField == null) {
hitField = new InternalSearchHitField(field.name(), new ArrayList<Object>(2));
hitContext.hit().fields().put(field.name(), hitField);
}
hitField.values().add(value);
}
}
}

View File

@ -0,0 +1,95 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.fetch.partial;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.search.SearchParseElement;
import org.elasticsearch.search.internal.SearchContext;
import java.util.ArrayList;
import java.util.List;
/**
* <pre>
* "partial_fields" : {
* "test1" : {
* "includes" : "doc['field_name'].value"
* },
* "test2" : {
* "excludes" : "..."
* }
* }
* </pre>
*/
public class PartialFieldsParseElement implements SearchParseElement {
@Override
public void parse(XContentParser parser, SearchContext context) throws Exception {
XContentParser.Token token;
String currentFieldName = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
String fieldName = currentFieldName;
List<String> includes = null;
List<String> excludes = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_ARRAY) {
if ("includes".equals(currentFieldName) || "include".equals(currentFieldName)) {
if (includes == null) {
includes = new ArrayList<String>(2);
}
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
includes.add(parser.text());
}
} else if ("excludes".equals(currentFieldName) || "exclude".equals(currentFieldName)) {
if (excludes == null) {
excludes = new ArrayList<String>(2);
}
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
excludes.add(parser.text());
}
}
} else if (token.isValue()) {
if ("include".equals(currentFieldName)) {
if (includes == null) {
includes = new ArrayList<String>(2);
}
includes.add(parser.text());
} else if ("exclude".equals(currentFieldName)) {
if (excludes == null) {
excludes = new ArrayList<String>(2);
}
excludes.add(parser.text());
}
}
}
PartialFieldsContext.PartialField field = new PartialFieldsContext.PartialField(fieldName,
includes == null ? Strings.EMPTY_ARRAY : includes.toArray(new String[includes.size()]),
excludes == null ? Strings.EMPTY_ARRAY : excludes.toArray(new String[excludes.size()]));
context.partialFields().add(field);
}
}
}
}

View File

@ -50,6 +50,7 @@ import org.elasticsearch.search.SearchShardTarget;
import org.elasticsearch.search.dfs.DfsSearchResult;
import org.elasticsearch.search.facet.SearchContextFacets;
import org.elasticsearch.search.fetch.FetchSearchResult;
import org.elasticsearch.search.fetch.partial.PartialFieldsContext;
import org.elasticsearch.search.fetch.script.ScriptFieldsContext;
import org.elasticsearch.search.highlight.SearchContextHighlight;
import org.elasticsearch.search.lookup.SearchLookup;
@ -117,6 +118,8 @@ public class SearchContext implements Releasable {
private boolean version = false; // by default, we don't return versions
private List<String> fieldNames;
private ScriptFieldsContext scriptFields;
private PartialFieldsContext partialFields;
private int from = -1;
@ -146,8 +149,6 @@ public class SearchContext implements Releasable {
private SearchContextHighlight highlight;
private ScriptFieldsContext scriptFields;
private SearchLookup searchLookup;
private boolean queryRewritten;
@ -278,6 +279,17 @@ public class SearchContext implements Releasable {
return this.scriptFields;
}
public boolean hasPartialFields() {
return partialFields != null;
}
public PartialFieldsContext partialFields() {
if (partialFields == null) {
partialFields = new PartialFieldsContext();
}
return this.partialFields;
}
public ContextIndexSearcher searcher() {
return this.searcher;
}

View File

@ -116,6 +116,10 @@ public class SourceLookup implements Map {
return XContentMapValues.extractRawValues(path, loadSourceIfNeeded());
}
public Object filter(String[] includes, String[] excludes) {
return XContentMapValues.filter(loadSourceIfNeeded(), includes, excludes);
}
public Object extractValue(String path) {
return XContentMapValues.extractValue(path, loadSourceIfNeeded());
}

View File

@ -21,12 +21,20 @@ package org.elasticsearch.test.integration.search.fields;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.test.integration.AbstractNodesTests;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.client.Requests.refreshRequest;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.hamcrest.MatcherAssert.assertThat;
@ -41,8 +49,8 @@ public class SearchFieldsTests extends AbstractNodesTests {
@BeforeClass
public void createNodes() throws Exception {
startNode("server1");
startNode("server2");
startNode("node1", settingsBuilder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0));
startNode("client1", settingsBuilder().put("node.client", true).build());
client = getClient();
}
@ -53,13 +61,14 @@ public class SearchFieldsTests extends AbstractNodesTests {
}
protected Client getClient() {
return client("server1");
return client("client1");
}
@Test
public void testStoredFields() throws Exception {
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().prepareCreate("test").execute().actionGet();
client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("field1").field("type", "string").field("store", "yes").endObject()
@ -112,4 +121,160 @@ public class SearchFieldsTests extends AbstractNodesTests {
assertThat(searchResponse.hits().getAt(0).fields().get("field1").value().toString(), equalTo("value1"));
assertThat(searchResponse.hits().getAt(0).fields().get("field3").value().toString(), equalTo("value3"));
}
@Test
public void testScriptDocAndFields() throws Exception {
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().prepareCreate("test").execute().actionGet();
client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("num1").field("type", "double").field("store", "yes").endObject()
.endObject().endObject().endObject().string();
client.admin().indices().preparePutMapping().setType("type1").setSource(mapping).execute().actionGet();
client.prepareIndex("test", "type1", "1")
.setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).field("date", "1970-01-01T00:00:00").endObject())
.execute().actionGet();
client.admin().indices().prepareFlush().execute().actionGet();
client.prepareIndex("test", "type1", "2")
.setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).field("date", "1970-01-01T00:00:25").endObject())
.execute().actionGet();
client.admin().indices().prepareFlush().execute().actionGet();
client.prepareIndex("test", "type1", "3")
.setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).field("date", "1970-01-01T00:02:00").endObject())
.execute().actionGet();
client.admin().indices().refresh(refreshRequest()).actionGet();
logger.info("running doc['num1'].value");
SearchResponse response = client.prepareSearch()
.setQuery(matchAllQuery())
.addSort("num1", SortOrder.ASC)
.addScriptField("sNum1", "doc['num1'].value")
.addScriptField("sNum1_field", "_fields['num1'].value")
.addScriptField("date1", "doc['date'].date.millis")
.execute().actionGet();
assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
assertThat(response.hits().totalHits(), equalTo(3l));
assertThat(response.hits().getAt(0).isSourceEmpty(), equalTo(true));
assertThat(response.hits().getAt(0).id(), equalTo("1"));
assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0));
assertThat((Double) response.hits().getAt(0).fields().get("sNum1_field").values().get(0), equalTo(1.0));
assertThat((Long) response.hits().getAt(0).fields().get("date1").values().get(0), equalTo(0l));
assertThat(response.hits().getAt(1).id(), equalTo("2"));
assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0));
assertThat((Double) response.hits().getAt(1).fields().get("sNum1_field").values().get(0), equalTo(2.0));
assertThat((Long) response.hits().getAt(1).fields().get("date1").values().get(0), equalTo(25000l));
assertThat(response.hits().getAt(2).id(), equalTo("3"));
assertThat((Double) response.hits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0));
assertThat((Double) response.hits().getAt(2).fields().get("sNum1_field").values().get(0), equalTo(3.0));
assertThat((Long) response.hits().getAt(2).fields().get("date1").values().get(0), equalTo(120000l));
logger.info("running doc['num1'].value * factor");
Map<String, Object> params = MapBuilder.<String, Object>newMapBuilder().put("factor", 2.0).map();
response = client.prepareSearch()
.setQuery(matchAllQuery())
.addSort("num1", SortOrder.ASC)
.addScriptField("sNum1", "doc['num1'].value * factor", params)
.execute().actionGet();
assertThat(response.hits().totalHits(), equalTo(3l));
assertThat(response.hits().getAt(0).id(), equalTo("1"));
assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0));
assertThat(response.hits().getAt(1).id(), equalTo("2"));
assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(4.0));
assertThat(response.hits().getAt(2).id(), equalTo("3"));
assertThat((Double) response.hits().getAt(2).fields().get("sNum1").values().get(0), equalTo(6.0));
}
@Test
public void testScriptFieldUsingSource() throws Exception {
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().prepareCreate("test").execute().actionGet();
client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
client.prepareIndex("test", "type1", "1")
.setSource(jsonBuilder().startObject()
.startObject("obj1").field("test", "something").endObject()
.startObject("obj2").startArray("arr2").value("arr_value1").value("arr_value2").endArray().endObject()
.startArray("arr3").startObject().field("arr3_field1", "arr3_value1").endObject().endArray()
.endObject())
.execute().actionGet();
client.admin().indices().refresh(refreshRequest()).actionGet();
SearchResponse response = client.prepareSearch()
.setQuery(matchAllQuery())
.addField("_source.obj1") // we also automatically detect _source in fields
.addScriptField("s_obj1", "_source.obj1")
.addScriptField("s_obj1_test", "_source.obj1.test")
.addScriptField("s_obj2", "_source.obj2")
.addScriptField("s_obj2_arr2", "_source.obj2.arr2")
.addScriptField("s_arr3", "_source.arr3")
.execute().actionGet();
assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
Map<String, Object> sObj1 = response.hits().getAt(0).field("_source.obj1").value();
assertThat(sObj1.get("test").toString(), equalTo("something"));
assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something"));
sObj1 = response.hits().getAt(0).field("s_obj1").value();
assertThat(sObj1.get("test").toString(), equalTo("something"));
assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something"));
Map<String, Object> sObj2 = response.hits().getAt(0).field("s_obj2").value();
List sObj2Arr2 = (List) sObj2.get("arr2");
assertThat(sObj2Arr2.size(), equalTo(2));
assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1"));
assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2"));
sObj2Arr2 = (List) response.hits().getAt(0).field("s_obj2_arr2").value();
assertThat(sObj2Arr2.size(), equalTo(2));
assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1"));
assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2"));
List sObj2Arr3 = (List) response.hits().getAt(0).field("s_arr3").value();
assertThat(((Map) sObj2Arr3.get(0)).get("arr3_field1").toString(), equalTo("arr3_value1"));
}
@Test
public void testPartialFields() throws Exception {
client.admin().indices().prepareDelete().execute().actionGet();
client.admin().indices().prepareCreate("test").execute().actionGet();
client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
client.prepareIndex("test", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject()
.field("field1", "value1")
.startObject("obj1")
.startArray("arr1")
.startObject().startObject("obj2").field("field2", "value21").endObject().endObject()
.startObject().startObject("obj2").field("field2", "value22").endObject().endObject()
.endArray()
.endObject()
.endObject())
.execute().actionGet();
client.admin().indices().prepareRefresh().execute().actionGet();
SearchResponse response = client.prepareSearch("test")
.addPartialField("partial1", "obj1.arr1.*", null)
.addPartialField("partial2", null, "obj1.*")
.execute().actionGet();
assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
Map<String, Object> partial1 = response.hits().getAt(0).field("partial1").value();
assertThat(partial1, notNullValue());
assertThat(partial1.containsKey("field1"), equalTo(false));
assertThat(partial1.containsKey("obj1"), equalTo(true));
assertThat(((Map) partial1.get("obj1")).get("arr1"), instanceOf(List.class));
Map<String, Object> partial2 = response.hits().getAt(0).field("partial2").value();
assertThat(partial2, notNullValue());
assertThat(partial2.containsKey("obj1"), equalTo(false));
assertThat(partial2.containsKey("field1"), equalTo(true));
}
}

View File

@ -1,192 +0,0 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.test.integration.search.scriptfield;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.test.integration.AbstractNodesTests;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.client.Requests.refreshRequest;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
/**
*
*/
@Test
public class ScriptFieldSearchTests extends AbstractNodesTests {
private Client client;
@BeforeClass
public void createNodes() throws Exception {
startNode("server1", settingsBuilder().put("number_of_shards", 1).put("number_of_replicas", 0));
startNode("client1", settingsBuilder().put("node.client", true).build());
client = getClient();
}
@AfterClass
public void closeNodes() {
client.close();
closeNode("client1");
closeAllNodes();
}
protected Client getClient() {
return client("client1");
}
@Test
public void testDocAndFields() throws Exception {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (Exception e) {
// its ok
}
client.admin().indices().prepareCreate("test").execute().actionGet();
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
.startObject("num1").field("type", "double").field("store", "yes").endObject()
.endObject().endObject().endObject().string();
client.admin().indices().preparePutMapping().setType("type1").setSource(mapping).execute().actionGet();
client.prepareIndex("test", "type1", "1")
.setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).field("date", "1970-01-01T00:00:00").endObject())
.execute().actionGet();
client.admin().indices().prepareFlush().execute().actionGet();
client.prepareIndex("test", "type1", "2")
.setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).field("date", "1970-01-01T00:00:25").endObject())
.execute().actionGet();
client.admin().indices().prepareFlush().execute().actionGet();
client.prepareIndex("test", "type1", "3")
.setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).field("date", "1970-01-01T00:02:00").endObject())
.execute().actionGet();
client.admin().indices().refresh(refreshRequest()).actionGet();
logger.info("running doc['num1'].value");
SearchResponse response = client.prepareSearch()
.setQuery(matchAllQuery())
.addSort("num1", SortOrder.ASC)
.addScriptField("sNum1", "doc['num1'].value")
.addScriptField("sNum1_field", "_fields['num1'].value")
.addScriptField("date1", "doc['date'].date.millis")
.execute().actionGet();
assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
assertThat(response.hits().totalHits(), equalTo(3l));
assertThat(response.hits().getAt(0).isSourceEmpty(), equalTo(true));
assertThat(response.hits().getAt(0).id(), equalTo("1"));
assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0));
assertThat((Double) response.hits().getAt(0).fields().get("sNum1_field").values().get(0), equalTo(1.0));
assertThat((Long) response.hits().getAt(0).fields().get("date1").values().get(0), equalTo(0l));
assertThat(response.hits().getAt(1).id(), equalTo("2"));
assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0));
assertThat((Double) response.hits().getAt(1).fields().get("sNum1_field").values().get(0), equalTo(2.0));
assertThat((Long) response.hits().getAt(1).fields().get("date1").values().get(0), equalTo(25000l));
assertThat(response.hits().getAt(2).id(), equalTo("3"));
assertThat((Double) response.hits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0));
assertThat((Double) response.hits().getAt(2).fields().get("sNum1_field").values().get(0), equalTo(3.0));
assertThat((Long) response.hits().getAt(2).fields().get("date1").values().get(0), equalTo(120000l));
logger.info("running doc['num1'].value * factor");
Map<String, Object> params = MapBuilder.<String, Object>newMapBuilder().put("factor", 2.0).map();
response = client.prepareSearch()
.setQuery(matchAllQuery())
.addSort("num1", SortOrder.ASC)
.addScriptField("sNum1", "doc['num1'].value * factor", params)
.execute().actionGet();
assertThat(response.hits().totalHits(), equalTo(3l));
assertThat(response.hits().getAt(0).id(), equalTo("1"));
assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0));
assertThat(response.hits().getAt(1).id(), equalTo("2"));
assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(4.0));
assertThat(response.hits().getAt(2).id(), equalTo("3"));
assertThat((Double) response.hits().getAt(2).fields().get("sNum1").values().get(0), equalTo(6.0));
}
@Test
public void testScriptFieldUsingSource() throws Exception {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (Exception e) {
// its ok
}
client.admin().indices().prepareCreate("test").execute().actionGet();
client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
client.prepareIndex("test", "type1", "1")
.setSource(jsonBuilder().startObject()
.startObject("obj1").field("test", "something").endObject()
.startObject("obj2").startArray("arr2").value("arr_value1").value("arr_value2").endArray().endObject()
.startArray("arr3").startObject().field("arr3_field1", "arr3_value1").endObject().endArray()
.endObject())
.execute().actionGet();
client.admin().indices().refresh(refreshRequest()).actionGet();
SearchResponse response = client.prepareSearch()
.setQuery(matchAllQuery())
.addField("_source.obj1") // we also automatically detect _source in fields
.addScriptField("s_obj1", "_source.obj1")
.addScriptField("s_obj1_test", "_source.obj1.test")
.addScriptField("s_obj2", "_source.obj2")
.addScriptField("s_obj2_arr2", "_source.obj2.arr2")
.addScriptField("s_arr3", "_source.arr3")
.execute().actionGet();
assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
Map<String, Object> sObj1 = (Map<String, Object>) response.hits().getAt(0).field("_source.obj1").value();
assertThat(sObj1.get("test").toString(), equalTo("something"));
assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something"));
sObj1 = (Map<String, Object>) response.hits().getAt(0).field("s_obj1").value();
assertThat(sObj1.get("test").toString(), equalTo("something"));
assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something"));
Map<String, Object> sObj2 = (Map<String, Object>) response.hits().getAt(0).field("s_obj2").value();
List sObj2Arr2 = (List) sObj2.get("arr2");
assertThat(sObj2Arr2.size(), equalTo(2));
assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1"));
assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2"));
sObj2Arr2 = (List) response.hits().getAt(0).field("s_obj2_arr2").value();
assertThat(sObj2Arr2.size(), equalTo(2));
assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1"));
assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2"));
List sObj2Arr3 = (List) response.hits().getAt(0).field("s_arr3").value();
assertThat(((Map) sObj2Arr3.get(0)).get("arr3_field1").toString(), equalTo("arr3_value1"));
}
}

View File

@ -80,6 +80,10 @@ public class XContentMapValuesTests {
filter = XContentMapValues.filter(source, new String[]{"test1*"}, Strings.EMPTY_ARRAY);
assertThat(filter.get("test1"), equalTo(source.get("test1")));
assertThat(filter.containsKey("path1"), equalTo(false));
filter = XContentMapValues.filter(source, new String[]{"path1.path2.*"}, Strings.EMPTY_ARRAY);
assertThat(filter.get("path1"), equalTo(source.get("path1")));
assertThat(filter.containsKey("test1"), equalTo(false));
}
@SuppressWarnings({"unchecked"})