Martijn van Groningen
parent
8235b89e9c
commit
bc0c7f8f28
|
|
@ -118,7 +118,7 @@ public class SimpleIdCache extends AbstractIndexComponent implements IdCache, Se
|
|||
Map<Object, IndexReader> cacheToReader = new HashMap<Object, IndexReader>();
|
||||
|
||||
// We don't want to load uid of child documents, this allows us to not load uids of child types.
|
||||
TreeSet<HashedBytesArray> parentTypes = new TreeSet<HashedBytesArray>(UTF8SortedAsUnicodeComparator.utf8SortedAsUnicodeSortOrder);
|
||||
NavigableSet<HashedBytesArray> parentTypes = new TreeSet<HashedBytesArray>(UTF8SortedAsUnicodeComparator.utf8SortedAsUnicodeSortOrder);
|
||||
for (String type : indexService.mapperService().types()) {
|
||||
ParentFieldMapper parentFieldMapper = indexService.mapperService().documentMapper(type).parentFieldMapper();
|
||||
if (parentFieldMapper != null) {
|
||||
|
|
|
|||
384
src/test/java/org/elasticsearch/test/unit/index/cache/id/SimpleIdCacheTests.java
vendored
Normal file
384
src/test/java/org/elasticsearch/test/unit/index/cache/id/SimpleIdCacheTests.java
vendored
Normal file
|
|
@ -0,0 +1,384 @@
|
|||
/*
|
||||
* Licensed to ElasticSearch and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. ElasticSearch licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.test.unit.index.cache.id;
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import org.apache.lucene.analysis.standard.StandardAnalyzer;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.document.StringField;
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.index.DirectoryReader;
|
||||
import org.apache.lucene.index.IndexWriter;
|
||||
import org.apache.lucene.index.IndexWriterConfig;
|
||||
import org.apache.lucene.store.RAMDirectory;
|
||||
import org.elasticsearch.ElasticSearchException;
|
||||
import org.elasticsearch.common.bytes.HashedBytesArray;
|
||||
import org.elasticsearch.common.collect.Tuple;
|
||||
import org.elasticsearch.common.inject.Injector;
|
||||
import org.elasticsearch.common.lucene.Lucene;
|
||||
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.IndexShardMissingException;
|
||||
import org.elasticsearch.index.aliases.IndexAliasesService;
|
||||
import org.elasticsearch.index.analysis.AnalysisService;
|
||||
import org.elasticsearch.index.cache.IndexCache;
|
||||
import org.elasticsearch.index.cache.id.IdReaderCache;
|
||||
import org.elasticsearch.index.cache.id.IdReaderTypeCache;
|
||||
import org.elasticsearch.index.cache.id.simple.SimpleIdCache;
|
||||
import org.elasticsearch.index.engine.IndexEngine;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldDataService;
|
||||
import org.elasticsearch.index.gateway.IndexGateway;
|
||||
import org.elasticsearch.index.mapper.MapperService;
|
||||
import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
|
||||
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
|
||||
import org.elasticsearch.index.percolator.PercolatorService;
|
||||
import org.elasticsearch.index.query.IndexQueryParserService;
|
||||
import org.elasticsearch.index.service.IndexService;
|
||||
import org.elasticsearch.index.settings.IndexSettingsService;
|
||||
import org.elasticsearch.index.shard.service.IndexShard;
|
||||
import org.elasticsearch.index.similarity.SimilarityService;
|
||||
import org.elasticsearch.index.store.IndexStore;
|
||||
import org.elasticsearch.test.unit.index.mapper.MapperTests;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
import static org.hamcrest.MatcherAssert.assertThat;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.nullValue;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class SimpleIdCacheTests {
|
||||
|
||||
@Test
|
||||
public void testRefresh() throws Exception {
|
||||
SimpleIdCache idCache = createSimpleIdCache(Tuple.tuple("child", "parent"));
|
||||
IndexWriter writer = createIndexWriter();
|
||||
// Begins with parent, ends with child docs
|
||||
writer.addDocument(doc("parent", "1"));
|
||||
writer.addDocument(childDoc("child", "1", "parent", "1"));
|
||||
writer.addDocument(childDoc("child", "2", "parent", "1"));
|
||||
writer.addDocument(childDoc("child", "3", "parent", "1"));
|
||||
writer.addDocument(childDoc("child", "4", "parent", "1"));
|
||||
writer.commit();
|
||||
|
||||
// Begins with child, ends with parent docs
|
||||
writer.addDocument(childDoc("child", "5", "parent", "2"));
|
||||
writer.addDocument(doc("parent", "2"));
|
||||
writer.addDocument(doc("parent", "3"));
|
||||
writer.addDocument(doc("parent", "4"));
|
||||
writer.addDocument(doc("parent", "5"));
|
||||
writer.commit();
|
||||
|
||||
// Begins with parent, child docs in the middle and ends with parent doc
|
||||
writer.addDocument(doc("parent", "6"));
|
||||
writer.addDocument(childDoc("child", "6", "parent", "6"));
|
||||
writer.addDocument(childDoc("child", "7", "parent", "6"));
|
||||
writer.addDocument(childDoc("child", "8", "parent", "5"));
|
||||
writer.addDocument(childDoc("child", "9", "parent", "4"));
|
||||
writer.addDocument(doc("parent", "7"));
|
||||
writer.commit();
|
||||
|
||||
// Garbage segment
|
||||
writer.addDocument(doc("zzz", "1"));
|
||||
writer.addDocument(doc("xxx", "2"));
|
||||
writer.addDocument(doc("aaa", "3"));
|
||||
writer.addDocument(doc("ccc", "4"));
|
||||
writer.addDocument(doc("parent", "8"));
|
||||
writer.commit();
|
||||
|
||||
writer.close();
|
||||
DirectoryReader topLevelReader = DirectoryReader.open(writer.getDirectory());
|
||||
List<AtomicReaderContext> leaves = topLevelReader.getContext().leaves();
|
||||
idCache.refresh(leaves);
|
||||
|
||||
// Verify simple id cache for segment 1
|
||||
IdReaderCache readerCache = idCache.reader(leaves.get(0).reader());
|
||||
assertThat(readerCache.type("child"), nullValue());
|
||||
IdReaderTypeCache typeCache = readerCache.type("parent");
|
||||
assertThat(typeCache.idByDoc(0).toUtf8(), equalTo("1"));
|
||||
assertThat(typeCache.idByDoc(1), nullValue());
|
||||
assertThat(typeCache.idByDoc(2), nullValue());
|
||||
assertThat(typeCache.idByDoc(3), nullValue());
|
||||
assertThat(typeCache.idByDoc(4), nullValue());
|
||||
|
||||
assertThat(typeCache.parentIdByDoc(0), nullValue());
|
||||
assertThat(typeCache.parentIdByDoc(1).toUtf8(), equalTo("1"));
|
||||
assertThat(typeCache.parentIdByDoc(2).toUtf8(), equalTo("1"));
|
||||
assertThat(typeCache.parentIdByDoc(3).toUtf8(), equalTo("1"));
|
||||
assertThat(typeCache.parentIdByDoc(4).toUtf8(), equalTo("1"));
|
||||
|
||||
assertThat(typeCache.docById(new HashedBytesArray("1")), equalTo(0));
|
||||
assertThat(typeCache.docById(new HashedBytesArray("2")), equalTo(-1));
|
||||
assertThat(typeCache.docById(new HashedBytesArray("3")), equalTo(-1));
|
||||
assertThat(typeCache.docById(new HashedBytesArray("4")), equalTo(-1));
|
||||
|
||||
// Verify simple id cache for segment 2
|
||||
readerCache = idCache.reader(leaves.get(1).reader());
|
||||
assertThat(readerCache.type("child"), nullValue());
|
||||
typeCache = readerCache.type("parent");
|
||||
assertThat(typeCache.idByDoc(0), nullValue());
|
||||
assertThat(typeCache.idByDoc(1).toUtf8(), equalTo("2"));
|
||||
assertThat(typeCache.idByDoc(2).toUtf8(), equalTo("3"));
|
||||
assertThat(typeCache.idByDoc(3).toUtf8(), equalTo("4"));
|
||||
assertThat(typeCache.idByDoc(4).toUtf8(), equalTo("5"));
|
||||
|
||||
assertThat(typeCache.parentIdByDoc(0).toUtf8(), equalTo("2"));
|
||||
assertThat(typeCache.parentIdByDoc(1), nullValue());
|
||||
assertThat(typeCache.parentIdByDoc(2), nullValue());
|
||||
assertThat(typeCache.parentIdByDoc(3), nullValue());
|
||||
assertThat(typeCache.parentIdByDoc(4), nullValue());
|
||||
|
||||
assertThat(typeCache.docById(new HashedBytesArray("2")), equalTo(1));
|
||||
assertThat(typeCache.docById(new HashedBytesArray("3")), equalTo(2));
|
||||
assertThat(typeCache.docById(new HashedBytesArray("4")), equalTo(3));
|
||||
assertThat(typeCache.docById(new HashedBytesArray("5")), equalTo(4));
|
||||
|
||||
// Verify simple id cache for segment 3
|
||||
readerCache = idCache.reader(leaves.get(2).reader());
|
||||
assertThat(readerCache.type("child"), nullValue());
|
||||
typeCache = readerCache.type("parent");
|
||||
assertThat(typeCache.idByDoc(0).toUtf8(), equalTo("6"));
|
||||
assertThat(typeCache.idByDoc(1), nullValue());
|
||||
assertThat(typeCache.idByDoc(2), nullValue());
|
||||
assertThat(typeCache.idByDoc(3), nullValue());
|
||||
assertThat(typeCache.idByDoc(4), nullValue());
|
||||
assertThat(typeCache.idByDoc(5).toUtf8(), equalTo("7"));
|
||||
|
||||
assertThat(typeCache.parentIdByDoc(0), nullValue());
|
||||
assertThat(typeCache.parentIdByDoc(1).toUtf8(), equalTo("6"));
|
||||
assertThat(typeCache.parentIdByDoc(2).toUtf8(), equalTo("6"));
|
||||
assertThat(typeCache.parentIdByDoc(3).toUtf8(), equalTo("5"));
|
||||
assertThat(typeCache.parentIdByDoc(4).toUtf8(), equalTo("4"));
|
||||
assertThat(typeCache.parentIdByDoc(5), nullValue());
|
||||
|
||||
assertThat(typeCache.docById(new HashedBytesArray("6")), equalTo(0));
|
||||
assertThat(typeCache.docById(new HashedBytesArray("7")), equalTo(5));
|
||||
|
||||
// Verify simple id cache for segment 4
|
||||
readerCache = idCache.reader(leaves.get(3).reader());
|
||||
assertThat(readerCache.type("child"), nullValue());
|
||||
typeCache = readerCache.type("parent");
|
||||
assertThat(typeCache.idByDoc(0), nullValue());
|
||||
assertThat(typeCache.idByDoc(1), nullValue());
|
||||
assertThat(typeCache.idByDoc(2), nullValue());
|
||||
assertThat(typeCache.idByDoc(3), nullValue());
|
||||
assertThat(typeCache.idByDoc(4).toUtf8(), equalTo("8"));
|
||||
|
||||
assertThat(typeCache.parentIdByDoc(0), nullValue());
|
||||
assertThat(typeCache.parentIdByDoc(1), nullValue());
|
||||
assertThat(typeCache.parentIdByDoc(2), nullValue());
|
||||
assertThat(typeCache.parentIdByDoc(3), nullValue());
|
||||
assertThat(typeCache.parentIdByDoc(4), nullValue());
|
||||
|
||||
assertThat(typeCache.docById(new HashedBytesArray("8")), equalTo(4));
|
||||
}
|
||||
|
||||
@Test(expectedExceptions = AssertionError.class)
|
||||
public void testRefresh_tripAssert() throws Exception {
|
||||
SimpleIdCache idCache = createSimpleIdCache(Tuple.tuple("child", "parent"));
|
||||
IndexWriter writer = createIndexWriter();
|
||||
// Begins with parent, ends with child docs
|
||||
writer.addDocument(doc("parent", "1"));
|
||||
writer.addDocument(childDoc("child", "1", "parent", "1"));
|
||||
writer.addDocument(childDoc("child", "2", "parent", "1"));
|
||||
writer.addDocument(childDoc("child", "3", "parent", "1"));
|
||||
writer.addDocument(childDoc("child", "4", "parent", "1"));
|
||||
// Doc like this should never end up in the index, just wanna trip an assert here!
|
||||
Document document = new Document();
|
||||
document.add(new StringField(UidFieldMapper.NAME, "parent", Field.Store.NO));
|
||||
writer.addDocument(document);
|
||||
writer.commit();
|
||||
|
||||
writer.close();
|
||||
DirectoryReader topLevelReader = DirectoryReader.open(writer.getDirectory());
|
||||
List<AtomicReaderContext> leaves = topLevelReader.getContext().leaves();
|
||||
idCache.refresh(leaves);
|
||||
}
|
||||
|
||||
private Document doc(String type, String id) {
|
||||
Document parent = new Document();
|
||||
parent.add(new StringField(UidFieldMapper.NAME, String.format("%s#%s", type, id), Field.Store.NO));
|
||||
return parent;
|
||||
}
|
||||
|
||||
private Document childDoc(String type, String id, String parentType, String parentId) {
|
||||
Document parent = new Document();
|
||||
parent.add(new StringField(UidFieldMapper.NAME, String.format("%s#%s", type, id), Field.Store.NO));
|
||||
parent.add(new StringField(ParentFieldMapper.NAME, String.format("%s#%s", parentType, parentId), Field.Store.NO));
|
||||
return parent;
|
||||
}
|
||||
|
||||
private SimpleIdCache createSimpleIdCache(Tuple<String, String>... documentTypes) throws IOException {
|
||||
Settings settings = ImmutableSettings.EMPTY;
|
||||
Index index = new Index("test");
|
||||
SimpleIdCache idCache = new SimpleIdCache(index, settings);
|
||||
MapperService mapperService = MapperTests.newMapperService();
|
||||
|
||||
for (Tuple<String, String> documentType : documentTypes) {
|
||||
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(documentType.v1())
|
||||
.startObject("_parent").field("type", documentType.v2()).endObject()
|
||||
.endObject().endObject().string();
|
||||
mapperService.merge(documentType.v1(), defaultMapping, true);
|
||||
}
|
||||
|
||||
idCache.setIndexService(new StubIndexService(mapperService));
|
||||
return idCache;
|
||||
}
|
||||
|
||||
private IndexWriter createIndexWriter() throws IOException {
|
||||
return new IndexWriter(new RAMDirectory(), new IndexWriterConfig(Lucene.VERSION, new StandardAnalyzer(Lucene.VERSION)));
|
||||
}
|
||||
|
||||
private class StubIndexService implements IndexService {
|
||||
|
||||
private final MapperService mapperService;
|
||||
|
||||
private StubIndexService(MapperService mapperService) {
|
||||
this.mapperService = mapperService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Injector injector() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexGateway gateway() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexCache cache() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexFieldDataService fieldData() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexSettingsService settingsService() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PercolatorService percolateService() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public AnalysisService analysisService() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public MapperService mapperService() {
|
||||
return mapperService;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexQueryParserService queryParserService() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public SimilarityService similarityService() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexAliasesService aliasesService() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexEngine engine() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexStore store() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexShard createShard(int sShardId) throws ElasticSearchException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void removeShard(int shardId, String reason) throws ElasticSearchException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public int numberOfShards() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ImmutableSet<Integer> shardIds() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasShard(int shardId) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexShard shard(int shardId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexShard shardSafe(int shardId) throws IndexShardMissingException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Injector shardInjector(int shardId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Injector shardInjectorSafe(int shardId) throws IndexShardMissingException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Index index() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<IndexShard> iterator() {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue