Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-4144: don't cache replicated index files 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1419980 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley 2012-12-11 02:58:08 +00:00
parent f4371dcedf
commit edca562dad
4 changed files with 23 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
View File

@ -21,6 +21,7 @@ import java.io.Closeable;
import java.io.IOException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
import org.apache.solr.common.SolrException;
import org.apache.solr.core.CachingDirectoryFactory.CloseListener;
@ -34,7 +35,13 @@ import org.slf4j.LoggerFactory;
*/
public abstract class DirectoryFactory implements NamedListInitializedPlugin,
Closeable {
// Estimate 10M docs, 100GB size, to avoid caching by NRTCachingDirectory
// Stayed away from upper bounds of the int/long in case any other code tried to aggregate these numbers.
// A large estimate should currently have no other side effects.
public static final IOContext IOCONTEXT_NO_CACHE = new IOContext(new FlushInfo(10*1000*1000, 100L*1000*1000*1000));
private static final Logger log = LoggerFactory.getLogger(DirectoryFactory.class.getName());
/**
@ -85,8 +92,8 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
*
* @throws IOException If there is a low-level I/O error.
*/
public void move(Directory fromDir, Directory toDir, String fileName) throws IOException {
fromDir.copy(toDir, fileName, fileName, IOContext.DEFAULT);
public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext) throws IOException {
fromDir.copy(toDir, fileName, fileName, ioContext);
fromDir.deleteFile(fileName);
}

6
solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java
View File

@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.commons.io.FileUtils;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IOContext;
/**
* Directory provider which mimics original Solr
@ -59,7 +60,8 @@ public class StandardDirectoryFactory extends CachingDirectoryFactory {
* @throws IOException
* If there is a low-level I/O error.
*/
public void move(Directory fromDir, Directory toDir, String fileName)
@Override
public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext)
throws IOException {
if (fromDir instanceof FSDirectory && toDir instanceof FSDirectory) {
File dir1 = ((FSDirectory) fromDir).getDirectory();
@ -72,6 +74,6 @@ public class StandardDirectoryFactory extends CachingDirectoryFactory {
}
}
super.move(fromDir, toDir, fileName);
super.move(fromDir, toDir, fileName, ioContext);
}
}

11
solr/core/src/java/org/apache/solr/handler/SnapPuller.java
View File

@ -88,6 +88,7 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CachingDirectoryFactory.CloseListener;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.IndexDeletionPolicyWrapper;
import org.apache.solr.core.NRTCachingDirectoryFactory;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.ReplicationHandler.FileInfo;
import org.apache.solr.request.LocalSolrQueryRequest;
@ -568,7 +569,7 @@ public class SnapPuller {
props.setProperty(REPLICATION_FAILED_AT_LIST, sb.toString());
}
final IndexOutput out = dir.createOutput(REPLICATION_PROPERTIES, IOContext.DEFAULT);
final IndexOutput out = dir.createOutput(REPLICATION_PROPERTIES, DirectoryFactory.IOCONTEXT_NO_CACHE);
OutputStream outFile = new PropertiesOutputStream(out);
try {
props.store(outFile, "Replication details");
@ -771,7 +772,7 @@ public class SnapPuller {
return false;
}
try {
solrCore.getDirectoryFactory().move(tmpIdxDir, indexDir, fname);
solrCore.getDirectoryFactory().move(tmpIdxDir, indexDir, fname, DirectoryFactory.IOCONTEXT_NO_CACHE);
success = true;
} catch (IOException e) {
SolrException.log(LOG, "Could not move file", e);
@ -844,7 +845,7 @@ public class SnapPuller {
try {
dir = solrCore.getDirectoryFactory().get(solrCore.getDataDir(), solrCore.getSolrConfig().indexConfig.lockType);
if (dir.fileExists("index.properties")){
final IndexInput input = dir.openInput("index.properties", IOContext.DEFAULT);
final IndexInput input = dir.openInput("index.properties", DirectoryFactory.IOCONTEXT_NO_CACHE);
final InputStream is = new PropertiesInputStream(input);
try {
@ -860,7 +861,7 @@ public class SnapPuller {
} catch (IOException e) {
// no problem
}
final IndexOutput out = dir.createOutput("index.properties", IOContext.DEFAULT);
final IndexOutput out = dir.createOutput("index.properties", DirectoryFactory.IOCONTEXT_NO_CACHE);
p.put("index", tmpIdxDirName);
OutputStream os = null;
try {
@ -1071,7 +1072,7 @@ public class SnapPuller {
indexGen = latestGen;
outStream = copy2Dir.createOutput(saveAs, IOContext.DEFAULT);
outStream = copy2Dir.createOutput(saveAs, DirectoryFactory.IOCONTEXT_NO_CACHE);
if (includeChecksum)
checksum = new Adler32();

5
solr/core/src/java/org/apache/solr/handler/SnapShooter.java
View File

@ -31,16 +31,17 @@ import java.util.regex.Pattern;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.SimpleFSLockFactory;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.SolrCore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* <p/> Provides functionality equivalent to the snapshooter script </p>
* This is no longer used in standard replication.
*
*
* @since solr 1.4
@ -200,7 +201,7 @@ public class SnapShooter {
throw new IOException(message);
}
sourceDir.copy(destDir, indexFile, indexFile, IOContext.DEFAULT);
sourceDir.copy(destDir, indexFile, indexFile, DirectoryFactory.IOCONTEXT_NO_CACHE);
}
}