From edca562dade00c1ce1c95de138ffe98aa6862df8 Mon Sep 17 00:00:00 2001 From: Yonik Seeley Date: Tue, 11 Dec 2012 02:58:08 +0000 Subject: [PATCH] SOLR-4144: don't cache replicated index files git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1419980 13f79535-47bb-0310-9956-ffa450edef68 --- .../java/org/apache/solr/core/DirectoryFactory.java | 13 ++++++++++--- .../apache/solr/core/StandardDirectoryFactory.java | 6 ++++-- .../java/org/apache/solr/handler/SnapPuller.java | 11 ++++++----- .../java/org/apache/solr/handler/SnapShooter.java | 5 +++-- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java index 6b5be744303..5db1ff380b2 100644 --- a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java @@ -21,6 +21,7 @@ import java.io.Closeable; import java.io.IOException; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FlushInfo; import org.apache.lucene.store.IOContext; import org.apache.solr.common.SolrException; import org.apache.solr.core.CachingDirectoryFactory.CloseListener; @@ -34,7 +35,13 @@ import org.slf4j.LoggerFactory; */ public abstract class DirectoryFactory implements NamedListInitializedPlugin, Closeable { - + + // Estimate 10M docs, 100GB size, to avoid caching by NRTCachingDirectory + // Stayed away from upper bounds of the int/long in case any other code tried to aggregate these numbers. + // A large estimate should currently have no other side effects. + public static final IOContext IOCONTEXT_NO_CACHE = new IOContext(new FlushInfo(10*1000*1000, 100L*1000*1000*1000)); + + private static final Logger log = LoggerFactory.getLogger(DirectoryFactory.class.getName()); /** @@ -85,8 +92,8 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin, * * @throws IOException If there is a low-level I/O error. */ - public void move(Directory fromDir, Directory toDir, String fileName) throws IOException { - fromDir.copy(toDir, fileName, fileName, IOContext.DEFAULT); + public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext) throws IOException { + fromDir.copy(toDir, fileName, fileName, ioContext); fromDir.deleteFile(fileName); } diff --git a/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java index 94e28f74eae..8c02c0cca77 100644 --- a/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.commons.io.FileUtils; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.IOContext; /** * Directory provider which mimics original Solr @@ -59,7 +60,8 @@ public class StandardDirectoryFactory extends CachingDirectoryFactory { * @throws IOException * If there is a low-level I/O error. */ - public void move(Directory fromDir, Directory toDir, String fileName) + @Override + public void move(Directory fromDir, Directory toDir, String fileName, IOContext ioContext) throws IOException { if (fromDir instanceof FSDirectory && toDir instanceof FSDirectory) { File dir1 = ((FSDirectory) fromDir).getDirectory(); @@ -72,6 +74,6 @@ public class StandardDirectoryFactory extends CachingDirectoryFactory { } } - super.move(fromDir, toDir, fileName); + super.move(fromDir, toDir, fileName, ioContext); } } diff --git a/solr/core/src/java/org/apache/solr/handler/SnapPuller.java b/solr/core/src/java/org/apache/solr/handler/SnapPuller.java index 8b9ef7bf8b8..66c1da9de2c 100644 --- a/solr/core/src/java/org/apache/solr/handler/SnapPuller.java +++ b/solr/core/src/java/org/apache/solr/handler/SnapPuller.java @@ -88,6 +88,7 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.core.CachingDirectoryFactory.CloseListener; import org.apache.solr.core.DirectoryFactory; import org.apache.solr.core.IndexDeletionPolicyWrapper; +import org.apache.solr.core.NRTCachingDirectoryFactory; import org.apache.solr.core.SolrCore; import org.apache.solr.handler.ReplicationHandler.FileInfo; import org.apache.solr.request.LocalSolrQueryRequest; @@ -568,7 +569,7 @@ public class SnapPuller { props.setProperty(REPLICATION_FAILED_AT_LIST, sb.toString()); } - final IndexOutput out = dir.createOutput(REPLICATION_PROPERTIES, IOContext.DEFAULT); + final IndexOutput out = dir.createOutput(REPLICATION_PROPERTIES, DirectoryFactory.IOCONTEXT_NO_CACHE); OutputStream outFile = new PropertiesOutputStream(out); try { props.store(outFile, "Replication details"); @@ -771,7 +772,7 @@ public class SnapPuller { return false; } try { - solrCore.getDirectoryFactory().move(tmpIdxDir, indexDir, fname); + solrCore.getDirectoryFactory().move(tmpIdxDir, indexDir, fname, DirectoryFactory.IOCONTEXT_NO_CACHE); success = true; } catch (IOException e) { SolrException.log(LOG, "Could not move file", e); @@ -844,7 +845,7 @@ public class SnapPuller { try { dir = solrCore.getDirectoryFactory().get(solrCore.getDataDir(), solrCore.getSolrConfig().indexConfig.lockType); if (dir.fileExists("index.properties")){ - final IndexInput input = dir.openInput("index.properties", IOContext.DEFAULT); + final IndexInput input = dir.openInput("index.properties", DirectoryFactory.IOCONTEXT_NO_CACHE); final InputStream is = new PropertiesInputStream(input); try { @@ -860,7 +861,7 @@ public class SnapPuller { } catch (IOException e) { // no problem } - final IndexOutput out = dir.createOutput("index.properties", IOContext.DEFAULT); + final IndexOutput out = dir.createOutput("index.properties", DirectoryFactory.IOCONTEXT_NO_CACHE); p.put("index", tmpIdxDirName); OutputStream os = null; try { @@ -1071,7 +1072,7 @@ public class SnapPuller { indexGen = latestGen; - outStream = copy2Dir.createOutput(saveAs, IOContext.DEFAULT); + outStream = copy2Dir.createOutput(saveAs, DirectoryFactory.IOCONTEXT_NO_CACHE); if (includeChecksum) checksum = new Adler32(); diff --git a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java index 0e50f3726ae..fec4fb133e0 100644 --- a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java +++ b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java @@ -31,16 +31,17 @@ import java.util.regex.Pattern; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.IOContext; import org.apache.lucene.store.Lock; import org.apache.lucene.store.SimpleFSLockFactory; import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.DirectoryFactory; import org.apache.solr.core.SolrCore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** *

Provides functionality equivalent to the snapshooter script

+ * This is no longer used in standard replication. * * * @since solr 1.4 @@ -200,7 +201,7 @@ public class SnapShooter { throw new IOException(message); } - sourceDir.copy(destDir, indexFile, indexFile, IOContext.DEFAULT); + sourceDir.copy(destDir, indexFile, indexFile, DirectoryFactory.IOCONTEXT_NO_CACHE); } }