honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enforce DirectoryReader when globals are loaded

This commit is contained in:
Simon Willnauer 2015-10-13 14:56:16 +02:00
parent d3436ff592
commit bd5ac9ce1a
21 changed files with 58 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchDirectoryReader.java
View File

@ -75,16 +75,9 @@ public final class ElasticsearchDirectoryReader extends FilterDirectoryReader {
}
@SuppressForbidden(reason = "This is the only sane way to add a ReaderClosedListener")
public static void addReaderCloseListener(IndexReader reader, IndexReader.ReaderClosedListener listener) {
public static void addReaderCloseListener(DirectoryReader reader, IndexReader.ReaderClosedListener listener) {
ElasticsearchDirectoryReader elasticsearchDirectoryReader = getElasticsearchDirectoryReader(reader);
if (elasticsearchDirectoryReader == null && reader instanceof LeafReader) {
ElasticsearchLeafReader leafReader = ElasticsearchLeafReader.getElasticsearchLeafReader((LeafReader) reader);
if (leafReader != null) {
assert reader.getCoreCacheKey() == leafReader.getCoreCacheKey();
leafReader.addReaderClosedListener(listener);
return;
}
} else {
if (elasticsearchDirectoryReader != null) {
assert reader.getCoreCacheKey() == elasticsearchDirectoryReader.getCoreCacheKey();
elasticsearchDirectoryReader.addReaderClosedListener(listener);
return;
@ -92,7 +85,7 @@ public final class ElasticsearchDirectoryReader extends FilterDirectoryReader {
throw new IllegalStateException("Can't install close listener reader is not an ElasticsearchDirectoryReader/ElasticsearchLeafReader");
}
public static ElasticsearchDirectoryReader getElasticsearchDirectoryReader(IndexReader reader) {
public static ElasticsearchDirectoryReader getElasticsearchDirectoryReader(DirectoryReader reader) {
if (reader instanceof FilterDirectoryReader) {
if (reader instanceof ElasticsearchDirectoryReader) {
return (ElasticsearchDirectoryReader) reader;

1
core/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.common.lucene.uid;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReader.CoreClosedListener;

6
core/src/main/java/org/elasticsearch/index/engine/Engine.java
View File

@ -579,10 +579,12 @@ public abstract class Engine implements Closeable {
private final String source;
private final IndexSearcher searcher;
private final DirectoryReader reader;
public Searcher(String source, IndexSearcher searcher) {
this.source = source;
this.searcher = searcher;
this.reader = (DirectoryReader) searcher.getIndexReader();
}
/**
@ -592,8 +594,8 @@ public abstract class Engine implements Closeable {
return source;
}
public IndexReader reader() {
return searcher.getIndexReader();
public DirectoryReader reader() {
return reader;
}
public IndexSearcher searcher() {

5
core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.DocIdSet;
@ -237,9 +238,9 @@ public interface IndexFieldData<FD extends AtomicFieldData> extends IndexCompone
public static interface Global<FD extends AtomicFieldData> extends IndexFieldData<FD> {
IndexFieldData<FD> loadGlobal(IndexReader indexReader);
IndexFieldData<FD> loadGlobal(DirectoryReader indexReader);
IndexFieldData<FD> localGlobalDirect(IndexReader indexReader) throws Exception;
IndexFieldData<FD> localGlobalDirect(DirectoryReader indexReader) throws Exception;
}

5
core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.util.Accountable;
@ -32,7 +33,7 @@ public interface IndexFieldDataCache {
<FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(LeafReaderContext context, IFD indexFieldData) throws Exception;
<FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(final IndexReader indexReader, final IFD indexFieldData) throws Exception;
<FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(final DirectoryReader indexReader, final IFD indexFieldData) throws Exception;
/**
* Clears all the field data stored cached in on this index.
@ -66,7 +67,7 @@ public interface IndexFieldDataCache {
@Override
@SuppressWarnings("unchecked")
public <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(IndexReader indexReader, IFD indexFieldData) throws Exception {
public <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(DirectoryReader indexReader, IFD indexFieldData) throws Exception {
return (IFD) indexFieldData.localGlobalDirect(indexReader);
}

5
core/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
@ -33,12 +34,12 @@ public interface IndexOrdinalsFieldData extends IndexFieldData.Global<AtomicOrdi
* potentially from a cache.
*/
@Override
IndexOrdinalsFieldData loadGlobal(IndexReader indexReader);
IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader);
/**
* Load a global view of the ordinals for the given {@link IndexReader}.
*/
@Override
IndexOrdinalsFieldData localGlobalDirect(IndexReader indexReader) throws Exception;
IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception;
}

5
core/src/main/java/org/elasticsearch/index/fielddata/IndexParentChildFieldData.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
@ -34,12 +35,12 @@ public interface IndexParentChildFieldData extends IndexFieldData.Global<AtomicP
* potentially from a cache.
*/
@Override
IndexParentChildFieldData loadGlobal(IndexReader indexReader);
IndexParentChildFieldData loadGlobal(DirectoryReader indexReader);
/**
* Load a global view of the ordinals for the given {@link IndexReader}.
*/
@Override
IndexParentChildFieldData localGlobalDirect(IndexReader indexReader) throws Exception;
IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception;
}

5
core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java
View File

@ -18,6 +18,7 @@
*/
package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.util.Accountable;
@ -59,12 +60,12 @@ public abstract class GlobalOrdinalsIndexFieldData extends AbstractIndexComponen
}
@Override
public IndexOrdinalsFieldData loadGlobal(IndexReader indexReader) {
public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
return this;
}
@Override
public IndexOrdinalsFieldData localGlobalDirect(IndexReader indexReader) throws Exception {
public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
return this;
}

4
core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
View File

@ -59,7 +59,7 @@ public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldD
}
@Override
public IndexOrdinalsFieldData loadGlobal(IndexReader indexReader) {
public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
if (indexReader.leaves().size() <= 1) {
// ordinals are already global
return this;
@ -76,7 +76,7 @@ public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldD
}
@Override
public IndexOrdinalsFieldData localGlobalDirect(IndexReader indexReader) throws Exception {
public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
return GlobalOrdinalsBuilder.build(indexReader, this, indexSettings, breakerService, logger);
}

10
core/src/main/java/org/elasticsearch/index/fielddata/plain/IndexIndexFieldData.java
View File

@ -19,11 +19,7 @@
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomAccessOrds;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.*;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.settings.Settings;
@ -123,12 +119,12 @@ public class IndexIndexFieldData extends AbstractIndexOrdinalsFieldData {
}
@Override
public IndexOrdinalsFieldData loadGlobal(IndexReader indexReader) {
public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
return this;
}
@Override
public IndexOrdinalsFieldData localGlobalDirect(IndexReader indexReader) throws Exception {
public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
return loadGlobal(indexReader);
}

8
core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java
View File

@ -135,7 +135,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare
}
@Override
public IndexParentChildFieldData loadGlobal(IndexReader indexReader) {
public IndexParentChildFieldData loadGlobal(DirectoryReader indexReader) {
if (indexReader.leaves().size() <= 1) {
// ordinals are already global
return this;
@ -170,7 +170,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare
}
@Override
public IndexParentChildFieldData localGlobalDirect(IndexReader indexReader) throws Exception {
public IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
final long startTime = System.nanoTime();
long ramBytesUsed = 0;
@ -347,7 +347,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare
}
@Override
public IndexParentChildFieldData loadGlobal(IndexReader indexReader) {
public IndexParentChildFieldData loadGlobal(DirectoryReader indexReader) {
if (indexReader.getCoreCacheKey() == reader.getCoreCacheKey()) {
return this;
}
@ -355,7 +355,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare
}
@Override
public IndexParentChildFieldData localGlobalDirect(IndexReader indexReader) throws Exception {
public IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
return loadGlobal(indexReader);
}

5
core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexReader;
import org.elasticsearch.ElasticsearchException;
@ -61,7 +62,7 @@ public class SortedSetDVOrdinalsIndexFieldData extends DocValuesIndexFieldData i
}
@Override
public IndexOrdinalsFieldData loadGlobal(IndexReader indexReader) {
public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
if (indexReader.leaves().size() <= 1) {
// ordinals are already global
return this;
@ -78,7 +79,7 @@ public class SortedSetDVOrdinalsIndexFieldData extends DocValuesIndexFieldData i
}
@Override
public IndexOrdinalsFieldData localGlobalDirect(IndexReader indexReader) throws Exception {
public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
return GlobalOrdinalsBuilder.build(indexReader, this, indexSettings, breakerService, logger);
}
}

3
core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
View File

@ -18,6 +18,7 @@
*/
package org.elasticsearch.index.query;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.search.IndexSearcher;
@ -290,7 +291,7 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
String joinField = ParentFieldMapper.joinField(parentType);
IndexSearcher indexSearcher = new IndexSearcher(reader);
indexSearcher.setQueryCache(null);
IndexParentChildFieldData indexParentChildFieldData = parentChildIndexFieldData.loadGlobal(indexSearcher.getIndexReader());
IndexParentChildFieldData indexParentChildFieldData = parentChildIndexFieldData.loadGlobal((DirectoryReader) reader);
MultiDocValues.OrdinalMap ordinalMap = ParentChildIndexFieldData.getOrdinalMap(indexParentChildFieldData, parentType);
return JoinUtil.createJoinQuery(joinField, innerQuery, toQuery, indexSearcher, scoreMode, ordinalMap, minChildren, maxChildren);
}

7
core/src/main/java/org/elasticsearch/index/shard/ShardUtils.java
View File

@ -47,15 +47,12 @@ public final class ShardUtils {
* will return null.
*/
@Nullable
public static ShardId extractShardId(IndexReader reader) {
public static ShardId extractShardId(DirectoryReader reader) {
final ElasticsearchDirectoryReader esReader = ElasticsearchDirectoryReader.getElasticsearchDirectoryReader(reader);
if (esReader != null) {
return esReader.shardId();
}
if (!reader.leaves().isEmpty()) {
return extractShardId(reader.leaves().get(0).reader());
}
return null;
throw new IllegalArgumentException("can't extract shard ID, can't unwrap ElasticsearchDirectoryReader");
}

3
core/src/main/java/org/elasticsearch/indices/IndicesWarmer.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.indices;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.metadata.IndexMetaData;
@ -176,7 +177,7 @@ public final class IndicesWarmer extends AbstractComponent {
return searcher;
}
public IndexReader reader() {
public DirectoryReader reader() {
return searcher.reader();
}

2
core/src/main/java/org/elasticsearch/indices/cache/request/IndicesRequestCache.java vendored
View File

@ -249,7 +249,7 @@ public class IndicesRequestCache extends AbstractComponent implements RemovalLis
if (!registeredClosedListeners.containsKey(cleanupKey)) {
Boolean previous = registeredClosedListeners.putIfAbsent(cleanupKey, Boolean.TRUE);
if (previous == null) {
ElasticsearchDirectoryReader.addReaderCloseListener(context.searcher().getIndexReader(), cleanupKey);
ElasticsearchDirectoryReader.addReaderCloseListener(context.searcher().getDirectoryReader(), cleanupKey);
}
}
} else {

3
core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java vendored
View File

@ -19,6 +19,7 @@
package org.elasticsearch.indices.fielddata.cache;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SegmentReader;
@ -168,7 +169,7 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
}
@Override
public <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(final IndexReader indexReader, final IFD indexFieldData) throws Exception {
public <FD extends AtomicFieldData, IFD extends IndexFieldData.Global<FD>> IFD load(final DirectoryReader indexReader, final IFD indexFieldData) throws Exception {
final ShardId shardId = ShardUtils.extractShardId(indexReader);
final Key key = new Key(this, indexReader.getCoreCacheKey(), shardId);
//noinspection unchecked

13
core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
View File

@ -18,12 +18,7 @@
*/
package org.elasticsearch.search.aggregations.support;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomAccessOrds;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.*;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.Bits;
@ -146,7 +141,7 @@ public abstract class ValuesSource {
@Override
public RandomAccessOrds globalOrdinalsValues(LeafReaderContext context) {
final IndexOrdinalsFieldData global = indexFieldData.loadGlobal(context.parent.reader());
final IndexOrdinalsFieldData global = indexFieldData.loadGlobal((DirectoryReader)context.parent.reader());
final AtomicOrdinalsFieldData atomicFieldData = global.load(context);
return atomicFieldData.getOrdinalsValues();
}
@ -162,7 +157,7 @@ public abstract class ValuesSource {
}
public long globalMaxOrd(IndexSearcher indexSearcher, String type) {
IndexReader indexReader = indexSearcher.getIndexReader();
DirectoryReader indexReader = (DirectoryReader) indexSearcher.getIndexReader();
if (indexReader.leaves().isEmpty()) {
return 0;
} else {
@ -175,7 +170,7 @@ public abstract class ValuesSource {
}
public SortedDocValues globalOrdinalsValues(String type, LeafReaderContext context) {
final IndexParentChildFieldData global = indexFieldData.loadGlobal(context.parent.reader());
final IndexParentChildFieldData global = indexFieldData.loadGlobal((DirectoryReader)context.parent.reader());
final AtomicParentChildFieldData atomicFieldData = global.load(context);
return atomicFieldData.getOrdinalsValues(type);
}

8
core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.search.internal;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
import org.apache.lucene.search.*;
@ -40,9 +41,12 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
private AggregatedDfs aggregatedDfs;
private final DirectoryReader directoryReader;
public ContextIndexSearcher(SearchContext searchContext, Engine.Searcher searcher) {
super(searcher.reader());
in = searcher.searcher();
directoryReader = searcher.reader();
setSimilarity(searcher.searcher().getSimilarity(true));
setQueryCache(searchContext.getQueryCache());
setQueryCachingPolicy(searchContext.indexShard().getQueryCachingPolicy());
@ -104,4 +108,8 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
}
return collectionStatistics;
}
public DirectoryReader getDirectoryReader() {
return directoryReader;
}
}

2
core/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
View File

@ -54,7 +54,7 @@ public abstract class AbstractFieldDataTestCase extends ESSingleNodeTestCase {
protected MapperService mapperService;
protected IndexWriter writer;
protected LeafReaderContext readerContext;
protected IndexReader topLevelReader;
protected DirectoryReader topLevelReader;
protected IndicesFieldDataCache indicesFieldDataCache;
protected abstract FieldDataType getFieldDataType();

13
core/src/test/java/org/elasticsearch/test/engine/AssertingSearcher.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.test.engine;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
import org.elasticsearch.common.logging.ESLogger;
@ -33,7 +34,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
class AssertingSearcher extends Engine.Searcher {
private final Engine.Searcher wrappedSearcher;
private final ShardId shardId;
private final IndexSearcher indexSearcher;
private RuntimeException firstReleaseStack;
private final Object lock = new Object();
private final int initialRefCount;
@ -50,7 +50,6 @@ class AssertingSearcher extends Engine.Searcher {
this.logger = logger;
this.shardId = shardId;
initialRefCount = wrappedSearcher.reader().getRefCount();
this.indexSearcher = indexSearcher;
assert initialRefCount > 0 : "IndexReader#getRefCount() was [" + initialRefCount + "] expected a value > [0] - reader is already closed";
}
@ -82,16 +81,6 @@ class AssertingSearcher extends Engine.Searcher {
}
}
@Override
public IndexReader reader() {
return indexSearcher.getIndexReader();
}
@Override
public IndexSearcher searcher() {
return indexSearcher;
}
public ShardId shardId() {
return shardId;
}