From a81f749ee1c4656d77cf25be7393b095a2e2a652 Mon Sep 17 00:00:00 2001 From: Mark Robert Miller Date: Tue, 11 Dec 2012 19:19:07 +0000 Subject: [PATCH] SOLR-4030: Allow rate limiting Directory IO based on the IO context. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1420336 13f79535-47bb-0310-9956-ffa450edef68 --- .../lucene/store/MockDirectoryWrapper.java | 4 ++ solr/CHANGES.txt | 3 ++ .../solr/core/CachingDirectoryFactory.java | 38 ++++++++++++++++++- .../solr/core/MMapDirectoryFactory.java | 1 + .../solr/core/NRTCachingDirectoryFactory.java | 1 + .../solr/collection1/conf/solrconfig.xml | 7 +++- .../apache/solr/BasicFunctionalityTest.java | 33 +++++++++++----- 7 files changed, 75 insertions(+), 12 deletions(-) diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java index 8d282b6589d..57ad4af16b1 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java @@ -120,6 +120,10 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper { this.lockFactory = new MockLockFactoryWrapper(this, delegate.getLockFactory()); init(); } + + public Directory getDelegate() { + return this.delegate; + } public int getInputCloneCount() { return inputCloneCount.get(); diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 2f9bc541379..886eef0ffb1 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -130,6 +130,9 @@ New Features * SOLR-3948: Calculate/display deleted documents in admin interface. (Shawn Heisey via Mark Miller) + +* SOLR-4030: Allow rate limiting Directory IO based on the IO context. + (Mark Miller, Radim Kolar) Optimizations ---------------------- diff --git a/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java index 8dea290b32a..b4e9eace9c3 100644 --- a/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java @@ -26,8 +26,10 @@ import java.util.Locale; import java.util.Map; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext.Context; import org.apache.lucene.store.NativeFSLockFactory; import org.apache.lucene.store.NoLockFactory; +import org.apache.lucene.store.RateLimitedDirectoryWrapper; import org.apache.lucene.store.SimpleFSLockFactory; import org.apache.lucene.store.SingleInstanceLockFactory; import org.apache.solr.common.SolrException; @@ -61,6 +63,14 @@ public abstract class CachingDirectoryFactory extends DirectoryFactory { protected Map byDirectoryCache = new HashMap(); protected Map> closeListeners = new HashMap>(); + + private Double maxWriteMBPerSecFlush; + + private Double maxWriteMBPerSecMerge; + + private Double maxWriteMBPerSecRead; + + private Double maxWriteMBPerSecDefault; public interface CloseListener { public void postClose(); @@ -233,6 +243,8 @@ public abstract class CachingDirectoryFactory extends DirectoryFactory { if (directory == null || forceNew) { directory = create(fullPath); + directory = rateLimit(directory); + CacheValue newCacheValue = new CacheValue(); newCacheValue.directory = directory; newCacheValue.path = fullPath; @@ -249,6 +261,25 @@ public abstract class CachingDirectoryFactory extends DirectoryFactory { return directory; } } + + private Directory rateLimit(Directory directory) { + if (maxWriteMBPerSecDefault != null || maxWriteMBPerSecFlush != null || maxWriteMBPerSecMerge != null || maxWriteMBPerSecRead != null) { + directory = new RateLimitedDirectoryWrapper(directory); + if (maxWriteMBPerSecDefault != null) { + ((RateLimitedDirectoryWrapper)directory).setMaxWriteMBPerSec(maxWriteMBPerSecDefault, Context.DEFAULT); + } + if (maxWriteMBPerSecFlush != null) { + ((RateLimitedDirectoryWrapper)directory).setMaxWriteMBPerSec(maxWriteMBPerSecFlush, Context.FLUSH); + } + if (maxWriteMBPerSecMerge != null) { + ((RateLimitedDirectoryWrapper)directory).setMaxWriteMBPerSec(maxWriteMBPerSecMerge, Context.MERGE); + } + if (maxWriteMBPerSecRead != null) { + ((RateLimitedDirectoryWrapper)directory).setMaxWriteMBPerSec(maxWriteMBPerSecRead, Context.READ); + } + } + return directory; + } /* * (non-Javadoc) @@ -270,7 +301,12 @@ public abstract class CachingDirectoryFactory extends DirectoryFactory { } @Override - public void init(NamedList args) {} + public void init(NamedList args) { + maxWriteMBPerSecFlush = (Double) args.get("maxWriteMBPerSecFlush"); + maxWriteMBPerSecMerge = (Double) args.get("maxWriteMBPerSecMerge"); + maxWriteMBPerSecRead = (Double) args.get("maxWriteMBPerSecRead"); + maxWriteMBPerSecDefault = (Double) args.get("maxWriteMBPerSecDefault"); + } /* * (non-Javadoc) diff --git a/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java index 1736a0378ba..58bb50cf87a 100644 --- a/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java @@ -46,6 +46,7 @@ public class MMapDirectoryFactory extends StandardDirectoryFactory { @Override public void init(NamedList args) { + super.init(args); SolrParams params = SolrParams.toSolrParams( args ); maxChunk = params.getInt("maxChunkSize", MMapDirectory.DEFAULT_MAX_BUFF); if (maxChunk <= 0){ diff --git a/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java index f97c2ee2925..79c3cfa9bf6 100644 --- a/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/NRTCachingDirectoryFactory.java @@ -35,6 +35,7 @@ public class NRTCachingDirectoryFactory extends StandardDirectoryFactory { @Override public void init(NamedList args) { + super.init(args); SolrParams params = SolrParams.toSolrParams(args); maxMergeSizeMB = params.getDouble("maxMergeSizeMB", 4); if (maxMergeSizeMB <= 0){ diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml index 206f47eba4d..71ada7afa4b 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml @@ -40,7 +40,12 @@ - + + 1000000 + 2000000 + 3000000 + 4000000 + ${tests.luceneMatchVersion:LUCENE_CURRENT} diff --git a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java index 9e4964d5803..38b725277ab 100644 --- a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java +++ b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java @@ -21,20 +21,22 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.StringWriter; import java.util.HashMap; -import java.util.Map; -import java.util.List; import java.util.LinkedList; +import java.util.List; +import java.util.Map; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; -import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LogMergePolicy; import org.apache.lucene.index.StorableField; import org.apache.lucene.index.StoredDocument; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext.Context; +import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.store.RateLimitedDirectoryWrapper; import org.apache.lucene.util.English; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; @@ -55,8 +57,6 @@ import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; import org.apache.solr.update.DirectUpdateHandler2; import org.apache.solr.util.RefCounted; - - import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; @@ -126,15 +126,28 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 { clearIndex(); // test merge factor picked up + // and for rate limited settings SolrCore core = h.getCore(); - RefCounted iw = ((DirectUpdateHandler2) core + RefCounted iwr = ((DirectUpdateHandler2) core .getUpdateHandler()).getSolrCoreState().getIndexWriter(core); try { - assertEquals("Mergefactor was not picked up", 8, ((LogMergePolicy) iw - .get().getConfig().getMergePolicy()).getMergeFactor()); + IndexWriter iw = iwr.get(); + assertEquals("Mergefactor was not picked up", 8, ((LogMergePolicy) iw.getConfig().getMergePolicy()).getMergeFactor()); + + Directory dir = iw.getDirectory(); + + if (dir instanceof MockDirectoryWrapper) { + dir = ((MockDirectoryWrapper)dir).getDelegate(); + } + + assertTrue(dir.getClass().getName(), dir instanceof RateLimitedDirectoryWrapper); + assertEquals(Double.valueOf(1000000), ((RateLimitedDirectoryWrapper)dir).getMaxWriteMBPerSec(Context.DEFAULT)); + assertEquals(Double.valueOf(2000000), ((RateLimitedDirectoryWrapper)dir).getMaxWriteMBPerSec(Context.FLUSH)); + assertEquals(Double.valueOf(3000000), ((RateLimitedDirectoryWrapper)dir).getMaxWriteMBPerSec(Context.MERGE)); + assertEquals(Double.valueOf(4000000), ((RateLimitedDirectoryWrapper)dir).getMaxWriteMBPerSec(Context.READ)); } finally { - iw.decref(); + iwr.decref(); } // test stats call NamedList stats = core.getStatistics();