honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-17 10:25:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

XFilteredQuery default strategy prefers query first in the deleted docs case

Browse Source 
Today we check if the DocIdSet we filter by is `fast` but the check fails
if the DocIdSet if wrapped in an `ApplyAcceptedDocsFilter` which is always
the case if the index has deleted documents. This commit unwraps
the original DocIdSet in the case of deleted documents.

Closes #6247
This commit is contained in:
Simon Willnauer 2014-05-21 11:30:43 +02:00
parent fa3bd738ab
commit f29744cc2f
3 changed files with 140 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
src/main/java/org/elasticsearch/common/lucene/search/ApplyAcceptedDocsFilter.java
View File

@ -53,7 +53,8 @@ public class ApplyAcceptedDocsFilter extends Filter {
// optimized wrapper for not deleted cases // optimized wrapper for not deleted cases
return new NotDeletedDocIdSet(docIdSet, acceptDocs); return new NotDeletedDocIdSet(docIdSet, acceptDocs);
} }
return BitsFilteredDocIdSet.wrap(docIdSet, acceptDocs); // we wrap this to make sure we can unwrap the inner docIDset in #unwrap
return new WrappedDocIdSet(BitsFilteredDocIdSet.wrap(docIdSet, acceptDocs), docIdSet);
} }
public Filter filter() { public Filter filter() {
@ -65,6 +66,15 @@ public class ApplyAcceptedDocsFilter extends Filter {
return filter.toString(); return filter.toString();
} }
public static DocIdSet unwrap(DocIdSet docIdSet) {
if (docIdSet instanceof NotDeletedDocIdSet) {
return ((NotDeletedDocIdSet) docIdSet).innerSet;
} else if (docIdSet instanceof WrappedDocIdSet) {
return ((WrappedDocIdSet) docIdSet).innerSet;
}
return docIdSet;
}
static class NotDeletedDocIdSet extends DocIdSet { static class NotDeletedDocIdSet extends DocIdSet {
private final DocIdSet innerSet; private final DocIdSet innerSet;
@ -167,4 +177,30 @@ public class ApplyAcceptedDocsFilter extends Filter {
return false; return false;
return true; return true;
} }
private static final class WrappedDocIdSet extends DocIdSet {
private final DocIdSet delegate;
private final DocIdSet innerSet;
private WrappedDocIdSet(DocIdSet delegate, DocIdSet innerSet) {
this.delegate = delegate;
this.innerSet = innerSet;
}
@Override
public DocIdSetIterator iterator() throws IOException {
return delegate.iterator();
}
@Override
public Bits bits() throws IOException {
return delegate.bits();
}
@Override
public boolean isCacheable() {
return delegate.isCacheable();
}
}
} }

2
src/main/java/org/elasticsearch/common/lucene/search/XFilteredQuery.java
View File

@ -225,7 +225,7 @@ public final class XFilteredQuery extends Query {
// CHANGE: handle "default" value // CHANGE: handle "default" value
if (threshold == -1) { if (threshold == -1) {
// default value, don't iterate on only apply filter after query if its not a "fast" docIdSet // default value, don't iterate on only apply filter after query if its not a "fast" docIdSet
if (!DocIdSets.isFastIterator(docIdSet)) { if (!DocIdSets.isFastIterator(ApplyAcceptedDocsFilter.unwrap(docIdSet))) {
return FilteredQuery.QUERY_FIRST_FILTER_STRATEGY.filteredScorer(context, weight, docIdSet); return FilteredQuery.QUERY_FIRST_FILTER_STRATEGY.filteredScorer(context, weight, docIdSet);
} }
} }

169
src/test/java/org/elasticsearch/search/query/SimpleQueryTests.java
View File

@ -47,8 +47,10 @@ import org.joda.time.format.ISODateTimeFormat;
import org.junit.Test; import org.junit.Test;
import java.io.IOException; import java.io.IOException;
import java.util.HashSet;
import java.util.Locale; import java.util.Locale;
import java.util.Random; import java.util.Random;
import java.util.Set;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
@ -141,7 +143,6 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
@Test @Test
public void passQueryAsStringTest() throws Exception { public void passQueryAsStringTest() throws Exception {
createIndex("test"); createIndex("test");
client().prepareIndex("test", "type1", "1").setSource("field1", "value1_1", "field2", "value2_1").setRefresh(true).get(); client().prepareIndex("test", "type1", "1").setSource("field1", "value1_1", "field2", "value2_1").setRefresh(true).get();
SearchResponse searchResponse = client().prepareSearch().setQuery("{ \"term\" : { \"field1\" : \"value1_1\" }}").get(); SearchResponse searchResponse = client().prepareSearch().setQuery("{ \"term\" : { \"field1\" : \"value1_1\" }}").get();
@ -152,10 +153,9 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
public void testIndexOptions() throws Exception { public void testIndexOptions() throws Exception {
assertAcked(prepareCreate("test") assertAcked(prepareCreate("test")
.addMapping("type1", "field1", "type=string,index_options=docs")); .addMapping("type1", "field1", "type=string,index_options=docs"));
indexRandom(true,
client().prepareIndex("test", "type1", "1").setSource("field1", "quick brown fox", "field2", "quick brown fox").get(); client().prepareIndex("test", "type1", "1").setSource("field1", "quick brown fox", "field2", "quick brown fox"),
client().prepareIndex("test", "type1", "2").setSource("field1", "quick lazy huge brown fox", "field2", "quick lazy huge brown fox").get(); client().prepareIndex("test", "type1", "2").setSource("field1", "quick lazy huge brown fox", "field2", "quick lazy huge brown fox"));
refresh();
SearchResponse searchResponse = client().prepareSearch().setQuery(matchQuery("field2", "quick brown").type(MatchQueryBuilder.Type.PHRASE).slop(0)).get(); SearchResponse searchResponse = client().prepareSearch().setQuery(matchQuery("field2", "quick brown").type(MatchQueryBuilder.Type.PHRASE).slop(0)).get();
assertHitCount(searchResponse, 1l); assertHitCount(searchResponse, 1l);
@ -931,11 +931,11 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
public void testFieldDataTermsFilter() throws Exception { public void testFieldDataTermsFilter() throws Exception {
assertAcked(prepareCreate("test").addMapping("type", "str", "type=string", "lng", "type=long", "dbl", "type=double")); assertAcked(prepareCreate("test").addMapping("type", "str", "type=string", "lng", "type=long", "dbl", "type=double"));
ensureGreen(); ensureGreen();
client().prepareIndex("test", "type", "1").setSource("str", "1", "lng", 1l, "dbl", 1.0d).get(); indexRandom(true,
client().prepareIndex("test", "type", "2").setSource("str", "2", "lng", 2l, "dbl", 2.0d).get(); client().prepareIndex("test", "type", "1").setSource("str", "1", "lng", 1l, "dbl", 1.0d),
client().prepareIndex("test", "type", "3").setSource("str", "3", "lng", 3l, "dbl", 3.0d).get(); client().prepareIndex("test", "type", "2").setSource("str", "2", "lng", 2l, "dbl", 2.0d),
client().prepareIndex("test", "type", "4").setSource("str", "4", "lng", 4l, "dbl", 4.0d).get(); client().prepareIndex("test", "type", "3").setSource("str", "3", "lng", 3l, "dbl", 3.0d),
refresh(); client().prepareIndex("test", "type", "4").setSource("str", "4", "lng", 4l, "dbl", 4.0d));
SearchResponse searchResponse = client().prepareSearch("test") SearchResponse searchResponse = client().prepareSearch("test")
.setQuery(filteredQuery(matchAllQuery(), termsFilter("str", "1", "4").execution("fielddata"))).get(); .setQuery(filteredQuery(matchAllQuery(), termsFilter("str", "1", "4").execution("fielddata"))).get();
@ -1734,10 +1734,10 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
.endObject().endObject().endObject())); .endObject().endObject().endObject()));
ensureGreen(); ensureGreen();
client().prepareIndex("simple", "lone").setId("1").setSource("text", "value1").get(); indexRandom(true,
client().prepareIndex("related", "parent").setId("2").setSource("text", "parent").get(); client().prepareIndex("simple", "lone").setId("1").setSource("text", "value1"),
client().prepareIndex("related", "child").setId("3").setParent("2").setSource("text", "value2").get(); client().prepareIndex("related", "parent").setId("2").setSource("text", "parent"),
refresh(); client().prepareIndex("related", "child").setId("3").setParent("2").setSource("text", "value2"));
//has_child fails if executed on "simple" index //has_child fails if executed on "simple" index
try { try {
@ -1759,18 +1759,17 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
} }
@Test @Test
public void testIndicesQueryMissingIndices() throws IOException { public void testIndicesQueryMissingIndices() throws IOException, ExecutionException, InterruptedException {
createIndex("index1"); createIndex("index1");
createIndex("index2"); createIndex("index2");
ensureGreen(); ensureGreen();
indexRandom(true,
client().prepareIndex("index1", "type1", "1").setSource("field", "match").get(); client().prepareIndex("index1", "type1", "1").setSource("field", "match"),
client().prepareIndex("index1", "type1", "2").setSource("field", "no_match").get(); client().prepareIndex("index1", "type1", "2").setSource("field", "no_match"),
client().prepareIndex("index2", "type1", "10").setSource("field", "match").get(); client().prepareIndex("index2", "type1", "10").setSource("field", "match"),
client().prepareIndex("index2", "type1", "20").setSource("field", "no_match").get(); client().prepareIndex("index2", "type1", "20").setSource("field", "no_match"),
client().prepareIndex("index3", "type1", "100").setSource("field", "match").get(); client().prepareIndex("index3", "type1", "100").setSource("field", "match"),
client().prepareIndex("index3", "type1", "200").setSource("field", "no_match").get(); client().prepareIndex("index3", "type1", "200").setSource("field", "no_match"));
refresh();
//all indices are missing //all indices are missing
SearchResponse searchResponse = client().prepareSearch().setQuery( SearchResponse searchResponse = client().prepareSearch().setQuery(
@ -1829,18 +1828,17 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
} }
@Test @Test
public void testIndicesFilterMissingIndices() throws IOException { public void testIndicesFilterMissingIndices() throws IOException, ExecutionException, InterruptedException {
createIndex("index1"); createIndex("index1");
createIndex("index2"); createIndex("index2");
ensureGreen(); ensureGreen();
indexRandom(true,
client().prepareIndex("index1", "type1", "1").setSource("field", "match").get(); client().prepareIndex("index1", "type1", "1").setSource("field", "match"),
client().prepareIndex("index1", "type1", "2").setSource("field", "no_match").get(); client().prepareIndex("index1", "type1", "2").setSource("field", "no_match"),
client().prepareIndex("index2", "type1", "10").setSource("field", "match").get(); client().prepareIndex("index2", "type1", "10").setSource("field", "match"),
client().prepareIndex("index2", "type1", "20").setSource("field", "no_match").get(); client().prepareIndex("index2", "type1", "20").setSource("field", "no_match"),
client().prepareIndex("index3", "type1", "100").setSource("field", "match").get(); client().prepareIndex("index3", "type1", "100").setSource("field", "match"),
client().prepareIndex("index3", "type1", "200").setSource("field", "no_match").get(); client().prepareIndex("index3", "type1", "200").setSource("field", "no_match"));
refresh();
//all indices are missing //all indices are missing
SearchResponse searchResponse = client().prepareSearch().setQuery( SearchResponse searchResponse = client().prepareSearch().setQuery(
@ -1902,15 +1900,14 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
} }
@Test @Test
public void testMinScore() { public void testMinScore() throws ExecutionException, InterruptedException {
createIndex("test"); createIndex("test");
ensureGreen(); ensureGreen();
indexRandom(true,
client().prepareIndex("test", "test", "1").setSource("score", 1.5).get(); client().prepareIndex("test", "test", "1").setSource("score", 1.5),
client().prepareIndex("test", "test", "2").setSource("score", 1).get(); client().prepareIndex("test", "test", "2").setSource("score", 1.0),
client().prepareIndex("test", "test", "3").setSource("score", 2).get(); client().prepareIndex("test", "test", "3").setSource("score", 2.0),
client().prepareIndex("test", "test", "4").setSource("score", 0.5).get(); client().prepareIndex("test", "test", "4").setSource("score", 0.5));
refresh();
SearchResponse searchResponse = client().prepareSearch("test").setQuery( SearchResponse searchResponse = client().prepareSearch("test").setQuery(
functionScoreQuery(scriptFunction("_doc['score'].value"))).setMinScore(1.5f).get(); functionScoreQuery(scriptFunction("_doc['score'].value"))).setMinScore(1.5f).get();
@ -1962,15 +1959,15 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
} }
@Test @Test
public void testSimpleQueryString() { public void testSimpleQueryString() throws ExecutionException, InterruptedException {
createIndex("test"); createIndex("test");
client().prepareIndex("test", "type1", "1").setSource("body", "foo").get(); indexRandom(true,
client().prepareIndex("test", "type1", "2").setSource("body", "bar").get(); client().prepareIndex("test", "type1", "1").setSource("body", "foo"),
client().prepareIndex("test", "type1", "3").setSource("body", "foo bar").get(); client().prepareIndex("test", "type1", "2").setSource("body", "bar"),
client().prepareIndex("test", "type1", "4").setSource("body", "quux baz eggplant").get(); client().prepareIndex("test", "type1", "3").setSource("body", "foo bar"),
client().prepareIndex("test", "type1", "5").setSource("body", "quux baz spaghetti").get(); client().prepareIndex("test", "type1", "4").setSource("body", "quux baz eggplant"),
client().prepareIndex("test", "type1", "6").setSource("otherbody", "spaghetti").get(); client().prepareIndex("test", "type1", "5").setSource("body", "quux baz spaghetti"),
refresh(); client().prepareIndex("test", "type1", "6").setSource("otherbody", "spaghetti"));
SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryString("foo bar")).get(); SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryString("foo bar")).get();
assertHitCount(searchResponse, 3l); assertHitCount(searchResponse, 3l);
@ -2083,15 +2080,15 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
} }
@Test @Test
public void testSimpleQueryStringFlags() { public void testSimpleQueryStringFlags() throws ExecutionException, InterruptedException {
createIndex("test"); createIndex("test");
client().prepareIndex("test", "type1", "1").setSource("body", "foo").get(); indexRandom(true,
client().prepareIndex("test", "type1", "2").setSource("body", "bar").get(); client().prepareIndex("test", "type1", "1").setSource("body", "foo"),
client().prepareIndex("test", "type1", "3").setSource("body", "foo bar").get(); client().prepareIndex("test", "type1", "2").setSource("body", "bar"),
client().prepareIndex("test", "type1", "4").setSource("body", "quux baz eggplant").get(); client().prepareIndex("test", "type1", "3").setSource("body", "foo bar"),
client().prepareIndex("test", "type1", "5").setSource("body", "quux baz spaghetti").get(); client().prepareIndex("test", "type1", "4").setSource("body", "quux baz eggplant"),
client().prepareIndex("test", "type1", "6").setSource("otherbody", "spaghetti").get(); client().prepareIndex("test", "type1", "5").setSource("body", "quux baz spaghetti"),
refresh(); client().prepareIndex("test", "type1", "6").setSource("otherbody", "spaghetti"));
SearchResponse searchResponse = client().prepareSearch().setQuery( SearchResponse searchResponse = client().prepareSearch().setQuery(
simpleQueryString("foo bar").flags(SimpleQueryStringFlag.ALL)).get(); simpleQueryString("foo bar").flags(SimpleQueryStringFlag.ALL)).get();
@ -2138,10 +2135,10 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
} }
@Test @Test
public void testSimpleQueryStringLenient() { public void testSimpleQueryStringLenient() throws ExecutionException, InterruptedException {
createIndex("test1", "test2"); createIndex("test1", "test2");
client().prepareIndex("test1", "type1", "1").setSource("field", "foo").get(); indexRandom(true, client().prepareIndex("test1", "type1", "1").setSource("field", "foo"),
client().prepareIndex("test2", "type1", "10").setSource("field", 5).get(); client().prepareIndex("test2", "type1", "10").setSource("field", 5));
refresh(); refresh();
SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryString("foo").field("field")).get(); SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryString("foo").field("field")).get();
@ -2156,13 +2153,12 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
} }
@Test @Test
public void testDateProvidedAsNumber() { public void testDateProvidedAsNumber() throws ExecutionException, InterruptedException {
createIndex("test"); createIndex("test");
assertAcked(client().admin().indices().preparePutMapping("test").setType("type").setSource("field", "type=date").get()); assertAcked(client().admin().indices().preparePutMapping("test").setType("type").setSource("field", "type=date").get());
client().prepareIndex("test", "type", "1").setSource("field", -1000000000001L).get(); indexRandom(true, client().prepareIndex("test", "type", "1").setSource("field", -1000000000001L),
client().prepareIndex("test", "type", "2").setSource("field", -1000000000000L).get(); client().prepareIndex("test", "type", "2").setSource("field", -1000000000000L),
client().prepareIndex("test", "type", "3").setSource("field", -999999999999L).get(); client().prepareIndex("test", "type", "3").setSource("field", -999999999999L));
refresh();
assertHitCount(client().prepareCount("test").setQuery(rangeQuery("field").lte(-1000000000000L)).get(), 2); assertHitCount(client().prepareCount("test").setQuery(rangeQuery("field").lte(-1000000000000L)).get(), 2);
assertHitCount(client().prepareCount("test").setQuery(rangeQuery("field").lte(-999999999999L)).get(), 3); assertHitCount(client().prepareCount("test").setQuery(rangeQuery("field").lte(-999999999999L)).get(), 3);
@ -2305,11 +2301,10 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
assertHitCount(searchResponse, 1l); assertHitCount(searchResponse, 1l);
} }
public void testMatchPhrasePrefixQuery() { public void testMatchPhrasePrefixQuery() throws ExecutionException, InterruptedException {
createIndex("test1"); createIndex("test1");
client().prepareIndex("test1", "type1", "1").setSource("field", "Johnnie Walker Black Label").get(); indexRandom(true, client().prepareIndex("test1", "type1", "1").setSource("field", "Johnnie Walker Black Label"),
client().prepareIndex("test1", "type1", "2").setSource("field", "trying out Elasticsearch").get(); client().prepareIndex("test1", "type1", "2").setSource("field", "trying out Elasticsearch"));
refresh();
SearchResponse searchResponse = client().prepareSearch().setQuery(matchQuery("field", "Johnnie la").slop(between(2,5)).type(Type.PHRASE_PREFIX)).get(); SearchResponse searchResponse = client().prepareSearch().setQuery(matchQuery("field", "Johnnie la").slop(between(2,5)).type(Type.PHRASE_PREFIX)).get();
assertHitCount(searchResponse, 1l); assertHitCount(searchResponse, 1l);
@ -2322,4 +2317,44 @@ public class SimpleQueryTests extends ElasticsearchIntegrationTest {
assertSearchHits(searchResponse, "2"); assertSearchHits(searchResponse, "2");
} }
@Test
public void testFilteredQuery() throws Exception {
ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder().put(indexSettings());
createIndex("test");
int numDocs = randomIntBetween(100, 150);
IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs];
for (int i = 0; i < numDocs; i++) {
docs[i] = client().prepareIndex("test", "type1", String.valueOf(i)).setSource("field1", English.intToEnglish(i));
}
indexRandom(true, docs);
ensureGreen();
int iters = between(1, 100);
for (int i = 0; i < iters; i++) {
String intToEnglish = English.intToEnglish(between(0, numDocs - 1));
String query = intToEnglish.split(" ")[0];
String filter = intToEnglish.split(" ")[0];
SearchResponse one = client().prepareSearch()
.setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.filteredQuery(QueryBuilders.termQuery("field1", query),
FilterBuilders.termFilter("field1", filter)))).setSize(numDocs).execute().actionGet();
SearchResponse other = client().prepareSearch()
.setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.filteredQuery(QueryBuilders.termQuery("field1", filter),
FilterBuilders.termFilter("field1", query)))).setSize(numDocs).execute().actionGet();
Set<String> oneIds = new HashSet<>();
for (SearchHit hit : one.getHits().hits()) {
oneIds.add(hit.id());
}
Set<String> otherIds = new HashSet<>();
for (SearchHit hit : other.getHits().hits()) {
otherIds.add(hit.id());
}
assertThat(oneIds.size(), equalTo(otherIds.size()));
for (String id : oneIds) {
assertThat(otherIds.contains(id), is(true));
}
}
}
} }