honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added simple id loading test. 

Relates to #3028
This commit is contained in:
Martijn van Groningen 2013-05-15 16:10:22 +02:00
parent 8235b89e9c
commit bc0c7f8f28
2 changed files with 385 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java vendored
View File

@ -118,7 +118,7 @@ public class SimpleIdCache extends AbstractIndexComponent implements IdCache, Se
Map<Object, IndexReader> cacheToReader = new HashMap<Object, IndexReader>(); Map<Object, IndexReader> cacheToReader = new HashMap<Object, IndexReader>();
// We don't want to load uid of child documents, this allows us to not load uids of child types. // We don't want to load uid of child documents, this allows us to not load uids of child types.
TreeSet<HashedBytesArray> parentTypes = new TreeSet<HashedBytesArray>(UTF8SortedAsUnicodeComparator.utf8SortedAsUnicodeSortOrder); NavigableSet<HashedBytesArray> parentTypes = new TreeSet<HashedBytesArray>(UTF8SortedAsUnicodeComparator.utf8SortedAsUnicodeSortOrder);
for (String type : indexService.mapperService().types()) { for (String type : indexService.mapperService().types()) {
ParentFieldMapper parentFieldMapper = indexService.mapperService().documentMapper(type).parentFieldMapper(); ParentFieldMapper parentFieldMapper = indexService.mapperService().documentMapper(type).parentFieldMapper();
if (parentFieldMapper != null) { if (parentFieldMapper != null) {

384
src/test/java/org/elasticsearch/test/unit/index/cache/id/SimpleIdCacheTests.java vendored Normal file
View File

@ -0,0 +1,384 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.test.unit.index.cache.id;
import com.google.common.collect.ImmutableSet;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.RAMDirectory;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.common.bytes.HashedBytesArray;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexShardMissingException;
import org.elasticsearch.index.aliases.IndexAliasesService;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.cache.IndexCache;
import org.elasticsearch.index.cache.id.IdReaderCache;
import org.elasticsearch.index.cache.id.IdReaderTypeCache;
import org.elasticsearch.index.cache.id.simple.SimpleIdCache;
import org.elasticsearch.index.engine.IndexEngine;
import org.elasticsearch.index.fielddata.IndexFieldDataService;
import org.elasticsearch.index.gateway.IndexGateway;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
import org.elasticsearch.index.mapper.internal.UidFieldMapper;
import org.elasticsearch.index.percolator.PercolatorService;
import org.elasticsearch.index.query.IndexQueryParserService;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.settings.IndexSettingsService;
import org.elasticsearch.index.shard.service.IndexShard;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.test.unit.index.mapper.MapperTests;
import org.testng.annotations.Test;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
/**
*/
public class SimpleIdCacheTests {
@Test
public void testRefresh() throws Exception {
SimpleIdCache idCache = createSimpleIdCache(Tuple.tuple("child", "parent"));
IndexWriter writer = createIndexWriter();
// Begins with parent, ends with child docs
writer.addDocument(doc("parent", "1"));
writer.addDocument(childDoc("child", "1", "parent", "1"));
writer.addDocument(childDoc("child", "2", "parent", "1"));
writer.addDocument(childDoc("child", "3", "parent", "1"));
writer.addDocument(childDoc("child", "4", "parent", "1"));
writer.commit();
// Begins with child, ends with parent docs
writer.addDocument(childDoc("child", "5", "parent", "2"));
writer.addDocument(doc("parent", "2"));
writer.addDocument(doc("parent", "3"));
writer.addDocument(doc("parent", "4"));
writer.addDocument(doc("parent", "5"));
writer.commit();
// Begins with parent, child docs in the middle and ends with parent doc
writer.addDocument(doc("parent", "6"));
writer.addDocument(childDoc("child", "6", "parent", "6"));
writer.addDocument(childDoc("child", "7", "parent", "6"));
writer.addDocument(childDoc("child", "8", "parent", "5"));
writer.addDocument(childDoc("child", "9", "parent", "4"));
writer.addDocument(doc("parent", "7"));
writer.commit();
// Garbage segment
writer.addDocument(doc("zzz", "1"));
writer.addDocument(doc("xxx", "2"));
writer.addDocument(doc("aaa", "3"));
writer.addDocument(doc("ccc", "4"));
writer.addDocument(doc("parent", "8"));
writer.commit();
writer.close();
DirectoryReader topLevelReader = DirectoryReader.open(writer.getDirectory());
List<AtomicReaderContext> leaves = topLevelReader.getContext().leaves();
idCache.refresh(leaves);
// Verify simple id cache for segment 1
IdReaderCache readerCache = idCache.reader(leaves.get(0).reader());
assertThat(readerCache.type("child"), nullValue());
IdReaderTypeCache typeCache = readerCache.type("parent");
assertThat(typeCache.idByDoc(0).toUtf8(), equalTo("1"));
assertThat(typeCache.idByDoc(1), nullValue());
assertThat(typeCache.idByDoc(2), nullValue());
assertThat(typeCache.idByDoc(3), nullValue());
assertThat(typeCache.idByDoc(4), nullValue());
assertThat(typeCache.parentIdByDoc(0), nullValue());
assertThat(typeCache.parentIdByDoc(1).toUtf8(), equalTo("1"));
assertThat(typeCache.parentIdByDoc(2).toUtf8(), equalTo("1"));
assertThat(typeCache.parentIdByDoc(3).toUtf8(), equalTo("1"));
assertThat(typeCache.parentIdByDoc(4).toUtf8(), equalTo("1"));
assertThat(typeCache.docById(new HashedBytesArray("1")), equalTo(0));
assertThat(typeCache.docById(new HashedBytesArray("2")), equalTo(-1));
assertThat(typeCache.docById(new HashedBytesArray("3")), equalTo(-1));
assertThat(typeCache.docById(new HashedBytesArray("4")), equalTo(-1));
// Verify simple id cache for segment 2
readerCache = idCache.reader(leaves.get(1).reader());
assertThat(readerCache.type("child"), nullValue());
typeCache = readerCache.type("parent");
assertThat(typeCache.idByDoc(0), nullValue());
assertThat(typeCache.idByDoc(1).toUtf8(), equalTo("2"));
assertThat(typeCache.idByDoc(2).toUtf8(), equalTo("3"));
assertThat(typeCache.idByDoc(3).toUtf8(), equalTo("4"));
assertThat(typeCache.idByDoc(4).toUtf8(), equalTo("5"));
assertThat(typeCache.parentIdByDoc(0).toUtf8(), equalTo("2"));
assertThat(typeCache.parentIdByDoc(1), nullValue());
assertThat(typeCache.parentIdByDoc(2), nullValue());
assertThat(typeCache.parentIdByDoc(3), nullValue());
assertThat(typeCache.parentIdByDoc(4), nullValue());
assertThat(typeCache.docById(new HashedBytesArray("2")), equalTo(1));
assertThat(typeCache.docById(new HashedBytesArray("3")), equalTo(2));
assertThat(typeCache.docById(new HashedBytesArray("4")), equalTo(3));
assertThat(typeCache.docById(new HashedBytesArray("5")), equalTo(4));
// Verify simple id cache for segment 3
readerCache = idCache.reader(leaves.get(2).reader());
assertThat(readerCache.type("child"), nullValue());
typeCache = readerCache.type("parent");
assertThat(typeCache.idByDoc(0).toUtf8(), equalTo("6"));
assertThat(typeCache.idByDoc(1), nullValue());
assertThat(typeCache.idByDoc(2), nullValue());
assertThat(typeCache.idByDoc(3), nullValue());
assertThat(typeCache.idByDoc(4), nullValue());
assertThat(typeCache.idByDoc(5).toUtf8(), equalTo("7"));
assertThat(typeCache.parentIdByDoc(0), nullValue());
assertThat(typeCache.parentIdByDoc(1).toUtf8(), equalTo("6"));
assertThat(typeCache.parentIdByDoc(2).toUtf8(), equalTo("6"));
assertThat(typeCache.parentIdByDoc(3).toUtf8(), equalTo("5"));
assertThat(typeCache.parentIdByDoc(4).toUtf8(), equalTo("4"));
assertThat(typeCache.parentIdByDoc(5), nullValue());
assertThat(typeCache.docById(new HashedBytesArray("6")), equalTo(0));
assertThat(typeCache.docById(new HashedBytesArray("7")), equalTo(5));
// Verify simple id cache for segment 4
readerCache = idCache.reader(leaves.get(3).reader());
assertThat(readerCache.type("child"), nullValue());
typeCache = readerCache.type("parent");
assertThat(typeCache.idByDoc(0), nullValue());
assertThat(typeCache.idByDoc(1), nullValue());
assertThat(typeCache.idByDoc(2), nullValue());
assertThat(typeCache.idByDoc(3), nullValue());
assertThat(typeCache.idByDoc(4).toUtf8(), equalTo("8"));
assertThat(typeCache.parentIdByDoc(0), nullValue());
assertThat(typeCache.parentIdByDoc(1), nullValue());
assertThat(typeCache.parentIdByDoc(2), nullValue());
assertThat(typeCache.parentIdByDoc(3), nullValue());
assertThat(typeCache.parentIdByDoc(4), nullValue());
assertThat(typeCache.docById(new HashedBytesArray("8")), equalTo(4));
}
@Test(expectedExceptions = AssertionError.class)
public void testRefresh_tripAssert() throws Exception {
SimpleIdCache idCache = createSimpleIdCache(Tuple.tuple("child", "parent"));
IndexWriter writer = createIndexWriter();
// Begins with parent, ends with child docs
writer.addDocument(doc("parent", "1"));
writer.addDocument(childDoc("child", "1", "parent", "1"));
writer.addDocument(childDoc("child", "2", "parent", "1"));
writer.addDocument(childDoc("child", "3", "parent", "1"));
writer.addDocument(childDoc("child", "4", "parent", "1"));
// Doc like this should never end up in the index, just wanna trip an assert here!
Document document = new Document();
document.add(new StringField(UidFieldMapper.NAME, "parent", Field.Store.NO));
writer.addDocument(document);
writer.commit();
writer.close();
DirectoryReader topLevelReader = DirectoryReader.open(writer.getDirectory());
List<AtomicReaderContext> leaves = topLevelReader.getContext().leaves();
idCache.refresh(leaves);
}
private Document doc(String type, String id) {
Document parent = new Document();
parent.add(new StringField(UidFieldMapper.NAME, String.format("%s#%s", type, id), Field.Store.NO));
return parent;
}
private Document childDoc(String type, String id, String parentType, String parentId) {
Document parent = new Document();
parent.add(new StringField(UidFieldMapper.NAME, String.format("%s#%s", type, id), Field.Store.NO));
parent.add(new StringField(ParentFieldMapper.NAME, String.format("%s#%s", parentType, parentId), Field.Store.NO));
return parent;
}
private SimpleIdCache createSimpleIdCache(Tuple<String, String>... documentTypes) throws IOException {
Settings settings = ImmutableSettings.EMPTY;
Index index = new Index("test");
SimpleIdCache idCache = new SimpleIdCache(index, settings);
MapperService mapperService = MapperTests.newMapperService();
for (Tuple<String, String> documentType : documentTypes) {
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(documentType.v1())
.startObject("_parent").field("type", documentType.v2()).endObject()
.endObject().endObject().string();
mapperService.merge(documentType.v1(), defaultMapping, true);
}
idCache.setIndexService(new StubIndexService(mapperService));
return idCache;
}
private IndexWriter createIndexWriter() throws IOException {
return new IndexWriter(new RAMDirectory(), new IndexWriterConfig(Lucene.VERSION, new StandardAnalyzer(Lucene.VERSION)));
}
private class StubIndexService implements IndexService {
private final MapperService mapperService;
private StubIndexService(MapperService mapperService) {
this.mapperService = mapperService;
}
@Override
public Injector injector() {
return null;
}
@Override
public IndexGateway gateway() {
return null;
}
@Override
public IndexCache cache() {
return null;
}
@Override
public IndexFieldDataService fieldData() {
return null;
}
@Override
public IndexSettingsService settingsService() {
return null;
}
@Override
public PercolatorService percolateService() {
return null;
}
@Override
public AnalysisService analysisService() {
return null;
}
@Override
public MapperService mapperService() {
return mapperService;
}
@Override
public IndexQueryParserService queryParserService() {
return null;
}
@Override
public SimilarityService similarityService() {
return null;
}
@Override
public IndexAliasesService aliasesService() {
return null;
}
@Override
public IndexEngine engine() {
return null;
}
@Override
public IndexStore store() {
return null;
}
@Override
public IndexShard createShard(int sShardId) throws ElasticSearchException {
return null;
}
@Override
public void removeShard(int shardId, String reason) throws ElasticSearchException {
}
@Override
public int numberOfShards() {
return 0;
}
@Override
public ImmutableSet<Integer> shardIds() {
return null;
}
@Override
public boolean hasShard(int shardId) {
return false;
}
@Override
public IndexShard shard(int shardId) {
return null;
}
@Override
public IndexShard shardSafe(int shardId) throws IndexShardMissingException {
return null;
}
@Override
public Injector shardInjector(int shardId) {
return null;
}
@Override
public Injector shardInjectorSafe(int shardId) throws IndexShardMissingException {
return null;
}
@Override
public Index index() {
return null;
}
@Override
public Iterator<IndexShard> iterator() {
return null;
}
}
}