From 688a6bd29b20ed148ccbf1bc03dcd9e3edb28fad Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 22 Nov 2013 18:36:36 +0100 Subject: [PATCH] Added random test for ParentQuery and ChildrenQuery. --- .../search/child/ChildrenQueryTests.java | 189 ++++++++++++++++++ .../index/search/child/MockScorer.java | 85 ++++++++ .../child/ParentConstantScoreQueryTests.java | 13 +- .../index/search/child/ParentQueryTests.java | 174 ++++++++++++++++ .../index/search/child/TestSearchContext.java | 8 +- 5 files changed, 460 insertions(+), 9 deletions(-) create mode 100644 src/test/java/org/elasticsearch/index/search/child/ChildrenQueryTests.java create mode 100644 src/test/java/org/elasticsearch/index/search/child/MockScorer.java create mode 100644 src/test/java/org/elasticsearch/index/search/child/ParentQueryTests.java diff --git a/src/test/java/org/elasticsearch/index/search/child/ChildrenQueryTests.java b/src/test/java/org/elasticsearch/index/search/child/ChildrenQueryTests.java new file mode 100644 index 00000000000..789590de408 --- /dev/null +++ b/src/test/java/org/elasticsearch/index/search/child/ChildrenQueryTests.java @@ -0,0 +1,189 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.elasticsearch.index.search.child; + +import com.carrotsearch.hppc.FloatArrayList; +import com.carrotsearch.hppc.ObjectObjectOpenHashMap; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.*; +import org.apache.lucene.queries.TermFilter; +import org.apache.lucene.search.*; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.FixedBitSet; +import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.mapper.Uid; +import org.elasticsearch.index.mapper.internal.ParentFieldMapper; +import org.elasticsearch.index.mapper.internal.TypeFieldMapper; +import org.elasticsearch.index.mapper.internal.UidFieldMapper; +import org.elasticsearch.search.internal.ContextIndexSearcher; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.test.ElasticsearchLuceneTestCase; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; + +import static org.elasticsearch.index.search.child.ChildrenConstantScoreQueryTests.assertBitSet; +import static org.elasticsearch.index.search.child.ChildrenConstantScoreQueryTests.createSearchContext; +import static org.hamcrest.Matchers.equalTo; + +public class ChildrenQueryTests extends ElasticsearchLuceneTestCase { + + @BeforeClass + public static void before() throws IOException { + SearchContext.setCurrent(createSearchContext("test", "parent", "child")); + } + + @AfterClass + public static void after() throws IOException { + SearchContext.removeCurrent(); + } + + @Test + public void testRandom() throws Exception { + Directory directory = newDirectory(); + RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); + int numUniqueChildValues = 1 + random().nextInt(TEST_NIGHTLY ? 6000 : 600); + String[] childValues = new String[numUniqueChildValues]; + for (int i = 0; i < numUniqueChildValues; i++) { + childValues[i] = Integer.toString(i); + } + + int childDocId = 0; + int numParentDocs = 1 + random().nextInt(TEST_NIGHTLY ? 20000 : 1000); + ObjectObjectOpenHashMap> childValueToParentIds = new ObjectObjectOpenHashMap>(); + for (int parentDocId = 0; parentDocId < numParentDocs; parentDocId++) { + boolean markParentAsDeleted = rarely(); + String parent = Integer.toString(parentDocId); + Document document = new Document(); + document.add(new StringField(UidFieldMapper.NAME, Uid.createUid("parent", parent), Field.Store.YES)); + document.add(new StringField(TypeFieldMapper.NAME, "parent", Field.Store.NO)); + if (markParentAsDeleted) { + document.add(new StringField("delete", "me", Field.Store.NO)); + } + indexWriter.addDocument(document); + + int numChildDocs = random().nextInt(TEST_NIGHTLY ? 100 : 25); + for (int i = 0; i < numChildDocs; i++) { + boolean markChildAsDeleted = rarely(); + String childValue = childValues[random().nextInt(childValues.length)]; + + document = new Document(); + document.add(new StringField(UidFieldMapper.NAME, Uid.createUid("child", Integer.toString(childDocId)), Field.Store.NO)); + document.add(new StringField(TypeFieldMapper.NAME, "child", Field.Store.NO)); + document.add(new StringField(ParentFieldMapper.NAME, Uid.createUid("parent", parent), Field.Store.NO)); + document.add(new StringField("field1", childValue, Field.Store.NO)); + if (markChildAsDeleted) { + document.add(new StringField("delete", "me", Field.Store.NO)); + } + indexWriter.addDocument(document); + + if (!markChildAsDeleted) { + NavigableMap parentIdToChildScores; + if (childValueToParentIds.containsKey(childValue)) { + parentIdToChildScores = childValueToParentIds.lget(); + } else { + childValueToParentIds.put(childValue, parentIdToChildScores = new TreeMap()); + } + if (!markParentAsDeleted) { + FloatArrayList childScores = parentIdToChildScores.get(parent); + if (childScores == null) { + parentIdToChildScores.put(parent, childScores = new FloatArrayList()); + } + childScores.add(1f); + } + } + } + } + + // Delete docs that are marked to be deleted. + indexWriter.deleteDocuments(new Term("delete", "me")); + + indexWriter.close(); + IndexReader indexReader = DirectoryReader.open(directory); + IndexSearcher searcher = new IndexSearcher(indexReader); + Engine.Searcher engineSearcher = new Engine.SimpleSearcher( + ChildrenQueryTests.class.getSimpleName(), searcher + ); + ((TestSearchContext) SearchContext.current()).setSearcher(new ContextIndexSearcher(SearchContext.current(), engineSearcher)); + + TermFilter parentFilter = new TermFilter(new Term(TypeFieldMapper.NAME, "parent")); + for (String childValue : childValues) { + Query childQuery = new ConstantScoreQuery(new TermQuery(new Term("field1", childValue))); + int shortCircuitParentDocSet = random().nextInt(numParentDocs); + ScoreType scoreType = ScoreType.values()[random().nextInt(ScoreType.values().length)]; + Query query = new ChildrenQuery("parent", "child", parentFilter, childQuery, scoreType, shortCircuitParentDocSet); + BitSetCollector collector = new BitSetCollector(indexReader.maxDoc()); + int numHits = 1 + random().nextInt(25); + TopScoreDocCollector actualTopDocsCollector = TopScoreDocCollector.create(numHits, false); + searcher.search(query, MultiCollector.wrap(collector, actualTopDocsCollector)); + FixedBitSet actualResult = collector.getResult(); + + FixedBitSet expectedResult = new FixedBitSet(indexReader.maxDoc()); + MockScorer mockScorer = new MockScorer(scoreType); + TopScoreDocCollector expectedTopDocsCollector = TopScoreDocCollector.create(numHits, false); + expectedTopDocsCollector.setScorer(mockScorer); + if (childValueToParentIds.containsKey(childValue)) { + AtomicReader slowAtomicReader = SlowCompositeReaderWrapper.wrap(indexReader); + Terms terms = slowAtomicReader.terms(UidFieldMapper.NAME); + if (terms != null) { + NavigableMap parentIdToChildScores = childValueToParentIds.lget(); + TermsEnum termsEnum = terms.iterator(null); + DocsEnum docsEnum = null; + for (Map.Entry entry : parentIdToChildScores.entrySet()) { + TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(Uid.createUidAsBytes("parent", entry.getKey())); + if (seekStatus == TermsEnum.SeekStatus.FOUND) { + docsEnum = termsEnum.docs(slowAtomicReader.getLiveDocs(), docsEnum, DocsEnum.FLAG_NONE); + expectedResult.set(docsEnum.nextDoc()); + mockScorer.scores = entry.getValue(); + expectedTopDocsCollector.collect(docsEnum.docID()); + } else if (seekStatus == TermsEnum.SeekStatus.END) { + break; + } + } + } + } + + assertBitSet(actualResult, expectedResult, searcher); + assertTopDocs(actualTopDocsCollector.topDocs(), expectedTopDocsCollector.topDocs()); + } + + indexReader.close(); + directory.close(); + } + + static void assertTopDocs(TopDocs actual, TopDocs expected) { + assertThat("actual.totalHits != expected.totalHits", actual.totalHits, equalTo(expected.totalHits)); + assertThat("actual.getMaxScore() != expected.getMaxScore()", actual.getMaxScore(), equalTo(expected.getMaxScore())); + assertThat("actual.scoreDocs.length != expected.scoreDocs.length", actual.scoreDocs.length, equalTo(actual.scoreDocs.length)); + for (int i = 0; i < actual.scoreDocs.length; i++) { + ScoreDoc actualHit = actual.scoreDocs[i]; + ScoreDoc expectedHit = expected.scoreDocs[i]; + assertThat("actualHit.doc != expectedHit.doc", actualHit.doc, equalTo(expectedHit.doc)); + assertThat("actualHit.score != expectedHit.score", actualHit.score, equalTo(expectedHit.score)); + } + } + +} diff --git a/src/test/java/org/elasticsearch/index/search/child/MockScorer.java b/src/test/java/org/elasticsearch/index/search/child/MockScorer.java new file mode 100644 index 00000000000..f0a24ae977e --- /dev/null +++ b/src/test/java/org/elasticsearch/index/search/child/MockScorer.java @@ -0,0 +1,85 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.elasticsearch.index.search.child; + +import com.carrotsearch.hppc.FloatArrayList; +import org.apache.lucene.search.Scorer; + +import java.io.IOException; + +class MockScorer extends Scorer { + + final ScoreType scoreType; + FloatArrayList scores; + + MockScorer(ScoreType scoreType) { + super(null); + this.scoreType = scoreType; + } + + @Override + public float score() throws IOException { + float aggregateScore = 0; + for (int i = 0; i < scores.elementsCount; i++) { + float score = scores.buffer[i]; + switch (scoreType) { + case MAX: + if (aggregateScore < score) { + aggregateScore = score; + } + break; + case SUM: + case AVG: + aggregateScore += score; + break; + } + } + + if (scoreType == ScoreType.AVG) { + aggregateScore /= scores.elementsCount; + } + + return aggregateScore; + } + + @Override + public int freq() throws IOException { + return 0; + } + + @Override + public int docID() { + return 0; + } + + @Override + public int nextDoc() throws IOException { + return 0; + } + + @Override + public int advance(int target) throws IOException { + return 0; + } + + @Override + public long cost() { + return 0; + } +} diff --git a/src/test/java/org/elasticsearch/index/search/child/ParentConstantScoreQueryTests.java b/src/test/java/org/elasticsearch/index/search/child/ParentConstantScoreQueryTests.java index 2d6a2d10146..eac9db46b9b 100644 --- a/src/test/java/org/elasticsearch/index/search/child/ParentConstantScoreQueryTests.java +++ b/src/test/java/org/elasticsearch/index/search/child/ParentConstantScoreQueryTests.java @@ -67,18 +67,15 @@ public class ParentConstantScoreQueryTests extends ElasticsearchLuceneTestCase { public void testRandom() throws Exception { Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); - - - int numUniqueChildValues = 1 + random().nextInt(TEST_NIGHTLY ? 20000 : 1000); - String[] parentValues = new String[numUniqueChildValues]; - for (int i = 0; i < numUniqueChildValues; i++) { + int numUniqueParentValues = 1 + random().nextInt(TEST_NIGHTLY ? 20000 : 1000); + String[] parentValues = new String[numUniqueParentValues]; + for (int i = 0; i < numUniqueParentValues; i++) { parentValues[i] = Integer.toString(i); } - ObjectObjectOpenHashMap> parentValueToChildDocIds = new ObjectObjectOpenHashMap>(); - int childDocId = 0; int numParentDocs = 1 + random().nextInt(TEST_NIGHTLY ? 10000 : 1000); + ObjectObjectOpenHashMap> parentValueToChildDocIds = new ObjectObjectOpenHashMap>(); for (int parentDocId = 0; parentDocId < numParentDocs; parentDocId++) { boolean markParentAsDeleted = rarely(); String parentValue = parentValues[random().nextInt(parentValues.length)]; @@ -127,7 +124,7 @@ public class ParentConstantScoreQueryTests extends ElasticsearchLuceneTestCase { IndexReader indexReader = DirectoryReader.open(directory); IndexSearcher searcher = new IndexSearcher(indexReader); Engine.Searcher engineSearcher = new Engine.SimpleSearcher( - ChildrenConstantScoreQueryTests.class.getSimpleName(), searcher + ParentConstantScoreQuery.class.getSimpleName(), searcher ); ((TestSearchContext) SearchContext.current()).setSearcher(new ContextIndexSearcher(SearchContext.current(), engineSearcher)); diff --git a/src/test/java/org/elasticsearch/index/search/child/ParentQueryTests.java b/src/test/java/org/elasticsearch/index/search/child/ParentQueryTests.java new file mode 100644 index 00000000000..4f74681ceec --- /dev/null +++ b/src/test/java/org/elasticsearch/index/search/child/ParentQueryTests.java @@ -0,0 +1,174 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.elasticsearch.index.search.child; + +import com.carrotsearch.hppc.FloatArrayList; +import com.carrotsearch.hppc.ObjectObjectOpenHashMap; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.*; +import org.apache.lucene.queries.TermFilter; +import org.apache.lucene.search.*; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.FixedBitSet; +import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.mapper.Uid; +import org.elasticsearch.index.mapper.internal.ParentFieldMapper; +import org.elasticsearch.index.mapper.internal.TypeFieldMapper; +import org.elasticsearch.index.mapper.internal.UidFieldMapper; +import org.elasticsearch.search.internal.ContextIndexSearcher; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.test.ElasticsearchLuceneTestCase; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; + +import static org.elasticsearch.index.search.child.ChildrenConstantScoreQueryTests.assertBitSet; +import static org.elasticsearch.index.search.child.ChildrenConstantScoreQueryTests.createSearchContext; +import static org.elasticsearch.index.search.child.ChildrenQueryTests.assertTopDocs; + +public class ParentQueryTests extends ElasticsearchLuceneTestCase { + + @BeforeClass + public static void before() throws IOException { + SearchContext.setCurrent(createSearchContext("test", "parent", "child")); + } + + @AfterClass + public static void after() throws IOException { + SearchContext.removeCurrent(); + } + + @Test + public void testRandom() throws Exception { + Directory directory = newDirectory(); + RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); + int numUniqueParentValues = 1 + random().nextInt(TEST_NIGHTLY ? 6000 : 600); + String[] parentValues = new String[numUniqueParentValues]; + for (int i = 0; i < numUniqueParentValues; i++) { + parentValues[i] = Integer.toString(i); + } + + int childDocId = 0; + int numParentDocs = 1 + random().nextInt(TEST_NIGHTLY ? 20000 : 1000); + ObjectObjectOpenHashMap> parentValueToChildIds = new ObjectObjectOpenHashMap>(); + for (int parentDocId = 0; parentDocId < numParentDocs; parentDocId++) { + boolean markParentAsDeleted = rarely(); + String parentValue = parentValues[random().nextInt(parentValues.length)]; + String parent = Integer.toString(parentDocId); + Document document = new Document(); + document.add(new StringField(UidFieldMapper.NAME, Uid.createUid("parent", parent), Field.Store.NO)); + document.add(new StringField(TypeFieldMapper.NAME, "parent", Field.Store.NO)); + document.add(new StringField("field1", parentValue, Field.Store.NO)); + if (markParentAsDeleted) { + document.add(new StringField("delete", "me", Field.Store.NO)); + } + indexWriter.addDocument(document); + + int numChildDocs = random().nextInt(TEST_NIGHTLY ? 100 : 25); + for (int i = 0; i < numChildDocs; i++) { + String child = Integer.toString(childDocId++); + boolean markChildAsDeleted = rarely(); + document = new Document(); + document.add(new StringField(UidFieldMapper.NAME, Uid.createUid("child", child), Field.Store.YES)); + document.add(new StringField(TypeFieldMapper.NAME, "child", Field.Store.NO)); + document.add(new StringField(ParentFieldMapper.NAME, Uid.createUid("parent", parent), Field.Store.NO)); + if (markChildAsDeleted) { + document.add(new StringField("delete", "me", Field.Store.NO)); + } + indexWriter.addDocument(document); + + if (!markChildAsDeleted) { + NavigableMap childIdToScore; + if (parentValueToChildIds.containsKey(parentValue)) { + childIdToScore = parentValueToChildIds.lget(); + } else { + parentValueToChildIds.put(parentValue, childIdToScore = new TreeMap()); + } + if (!markParentAsDeleted) { + assert !childIdToScore.containsKey(child); + childIdToScore.put(child, 1f); + } + } + } + } + + // Delete docs that are marked to be deleted. + indexWriter.deleteDocuments(new Term("delete", "me")); + + indexWriter.close(); + IndexReader indexReader = DirectoryReader.open(directory); + IndexSearcher searcher = new IndexSearcher(indexReader); + Engine.Searcher engineSearcher = new Engine.SimpleSearcher( + ParentQueryTests.class.getSimpleName(), searcher + ); + ((TestSearchContext) SearchContext.current()).setSearcher(new ContextIndexSearcher(SearchContext.current(), engineSearcher)); + + TermFilter childFilter = new TermFilter(new Term(TypeFieldMapper.NAME, "child")); + for (String parentValue : parentValues) { + Query parentQuery = new ConstantScoreQuery(new TermQuery(new Term("field1", parentValue))); + Query query = new ParentQuery(parentQuery,"parent", childFilter); + BitSetCollector collector = new BitSetCollector(indexReader.maxDoc()); + int numHits = 1 + random().nextInt(25); + TopScoreDocCollector actualTopDocsCollector = TopScoreDocCollector.create(numHits, false); + searcher.search(query, MultiCollector.wrap(collector, actualTopDocsCollector)); + FixedBitSet actualResult = collector.getResult(); + + FixedBitSet expectedResult = new FixedBitSet(indexReader.maxDoc()); + MockScorer mockScorer = new MockScorer(ScoreType.MAX); // just save one score per parent... + mockScorer.scores = new FloatArrayList(); + TopScoreDocCollector expectedTopDocsCollector = TopScoreDocCollector.create(numHits, false); + expectedTopDocsCollector.setScorer(mockScorer); + if (parentValueToChildIds.containsKey(parentValue)) { + AtomicReader slowAtomicReader = SlowCompositeReaderWrapper.wrap(indexReader); + Terms terms = slowAtomicReader.terms(UidFieldMapper.NAME); + if (terms != null) { + NavigableMap childIdsAndScore = parentValueToChildIds.lget(); + TermsEnum termsEnum = terms.iterator(null); + DocsEnum docsEnum = null; + for (Map.Entry entry : childIdsAndScore.entrySet()) { + TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(Uid.createUidAsBytes("child", entry.getKey())); + if (seekStatus == TermsEnum.SeekStatus.FOUND) { + docsEnum = termsEnum.docs(slowAtomicReader.getLiveDocs(), docsEnum, DocsEnum.FLAG_NONE); + expectedResult.set(docsEnum.nextDoc()); + mockScorer.scores.add(entry.getValue()); + expectedTopDocsCollector.collect(docsEnum.docID()); + mockScorer.scores.clear(); + } else if (seekStatus == TermsEnum.SeekStatus.END) { + break; + } + } + } + } + + assertBitSet(actualResult, expectedResult, searcher); + assertTopDocs(actualTopDocsCollector.topDocs(), expectedTopDocsCollector.topDocs()); + } + + indexReader.close(); + directory.close(); + } + +} diff --git a/src/test/java/org/elasticsearch/index/search/child/TestSearchContext.java b/src/test/java/org/elasticsearch/index/search/child/TestSearchContext.java index 889c0e9085c..db5c3e4c8b3 100644 --- a/src/test/java/org/elasticsearch/index/search/child/TestSearchContext.java +++ b/src/test/java/org/elasticsearch/index/search/child/TestSearchContext.java @@ -67,6 +67,7 @@ class TestSearchContext extends SearchContext { final IndexService indexService; ContextIndexSearcher searcher; + int size; TestSearchContext(CacheRecycler cacheRecycler, IdCache idCache, IndexService indexService) { this.cacheRecycler = cacheRecycler; @@ -390,9 +391,14 @@ class TestSearchContext extends SearchContext { @Override public int size() { - return 0; + return size; } + public void setSize(int size) { + this.size = size; + } + + @Override public SearchContext size(int size) { return null;