honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[CORE] Read segment info from latest commit whenever possible 

Instead of listing the directory to file the latest segments_N file, we
should re-use the generation/filename from the last commit. This allows
us to avoid potential race conditions on the filesystem as well as
reduce the number of directory listings performed.
This commit is contained in:
Lee Hinman 2015-05-26 14:33:24 -06:00
parent fcccd45601
commit 6646881bb6
5 changed files with 23 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/main/java/org/elasticsearch/common/lucene/Lucene.java
View File

@ -159,8 +159,11 @@ public class Lucene {
/**
* Reads the segments infos from the given commit, failing if it fails to load
*/
public static SegmentInfos readSegmentInfos(IndexCommit commit, Directory directory) throws IOException {
return SegmentInfos.readCommit(directory, commit.getSegmentsFileName());
public static SegmentInfos readSegmentInfos(IndexCommit commit) throws IOException {
// Using commit.getSegmentsFileName() does NOT work here, have to
// manually create the segment filename
String filename = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", commit.getGeneration());
return SegmentInfos.readCommit(commit.getDirectory(), filename);
}
/**

23
src/main/java/org/elasticsearch/index/engine/Engine.java
View File

@ -21,15 +21,7 @@ package org.elasticsearch.index.engine;
import com.google.common.base.Preconditions;
import org.apache.lucene.index.FilterLeafReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SegmentCommitInfo;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.*;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SearcherManager;
@ -324,7 +316,18 @@ public abstract class Engine implements Closeable {
return new CommitStats(getLastCommittedSegmentInfos());
}
/**
* Read the last segments info from the commit pointed to by the searcher manager
*/
protected static SegmentInfos readLastCommittedSegmentInfos(SearcherManager sm) throws IOException {
IndexSearcher searcher = sm.acquire();
try {
IndexCommit latestCommit = ((DirectoryReader) searcher.getIndexReader()).getIndexCommit();
return Lucene.readSegmentInfos(latestCommit);
} finally {
sm.release(searcher);
}
}
/**
* Global stats on segments.

2
src/main/java/org/elasticsearch/index/engine/InternalEngine.java
View File

@ -275,7 +275,7 @@ public class InternalEngine extends Engine {
try {
final DirectoryReader directoryReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(indexWriter, true), shardId);
searcherManager = new SearcherManager(directoryReader, searcherFactory);
lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
lastCommittedSegmentInfos = readLastCommittedSegmentInfos(searcherManager);
success = true;
return searcherManager;
} catch (IOException e) {

4
src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
View File

@ -79,7 +79,7 @@ public class ShadowEngine extends Engine {
if (Lucene.waitForIndex(store.directory(), nonexistentRetryTime)) {
reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(store.directory()), shardId);
this.searcherManager = new SearcherManager(reader, searcherFactory);
this.lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
this.lastCommittedSegmentInfos = readLastCommittedSegmentInfos(searcherManager);
success = true;
} else {
throw new IndexShardException(shardId, "failed to open a shadow engine after" +
@ -148,7 +148,7 @@ public class ShadowEngine extends Engine {
store.incRef();
try (ReleasableLock lock = readLock.acquire()) {
// reread the last committed segment infos
lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
lastCommittedSegmentInfos = readLastCommittedSegmentInfos(searcherManager);
} catch (Throwable e) {
if (isClosed.get() == false) {
logger.warn("failed to read latest segment infos on flush", e);

3
src/main/java/org/elasticsearch/index/store/Store.java
View File

@ -151,8 +151,9 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
* @throws IOException if the index is corrupted or the segments file is not present
*/
private static SegmentInfos readSegmentsInfo(IndexCommit commit, Directory directory) throws IOException {
assert commit == null || commit.getDirectory() == directory;
try {
return commit == null ? Lucene.readSegmentInfos(directory) : Lucene.readSegmentInfos(commit, directory);
return commit == null ? Lucene.readSegmentInfos(directory) : Lucene.readSegmentInfos(commit);
} catch (EOFException eof) {
// TODO this should be caught by lucene - EOF is almost certainly an index corruption
throw new CorruptIndexException("Read past EOF while reading segment infos", "commit(" + commit + ")", eof);