diff --git a/docs/content/development/extensions-core/lookups-cached-global.md b/docs/content/development/extensions-core/lookups-cached-global.md index 7199e76ded2..5198b9787b5 100644 --- a/docs/content/development/extensions-core/lookups-cached-global.md +++ b/docs/content/development/extensions-core/lookups-cached-global.md @@ -147,13 +147,16 @@ setting namespaces (broker, peon, historical) |--------|-----------|-------| |`druid.lookup.namespace.cache.type`|Specifies the type of caching to be used by the namespaces. May be one of [`offHeap`, `onHeap`]. `offHeap` uses a temporary file for off-heap storage of the namespace (memory mapped files). `onHeap` stores all cache on the heap in standard java map types.|`onHeap`| |`druid.lookup.namespace.numExtractionThreads`|The number of threads in the thread pool dedicated for lookup extraction and updates. This number may need to be scaled up, if you have a lot of lookups and they take long time to extract, to avoid timeouts.|2| +|`druid.lookup.namespace.numBufferedEntries`|If using offHeap caching, the number of records to be stored on an on-heap buffer.|100,000| The cache is populated in different ways depending on the settings below. In general, most namespaces employ a `pollPeriod` at the end of which time they poll the remote resource of interest for updates. `onHeap` uses `ConcurrentMap`s in the java heap, and thus affects garbage collection and heap sizing. -`offHeap` uses a 10MB on-heap buffer and MapDB using memory-mapped files in the java temporary directory. -So if total `cachedNamespace` lookup size is in excess of 10MB, the extra will be kept in memory as page cache, and paged in and out by general OS tunings. +`offHeap` uses an on-heap buffer and MapDB using memory-mapped files in the java temporary directory. +So if total number of entries in the `cachedNamespace` is in excess of the buffer's configured capacity, the extra will be kept in memory as page cache, and paged in and out by general OS tunings. +It's highly recommended that `druid.lookup.namespace.numBufferedEntries` is set when using `offHeap`, the value should be chosen from the range between 10% and 50% of the number of entries in the lookup. + # Supported Lookups diff --git a/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/NamespaceExtractionConfig.java b/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/NamespaceExtractionConfig.java index b04af0688b9..7712b34243c 100644 --- a/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/NamespaceExtractionConfig.java +++ b/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/NamespaceExtractionConfig.java @@ -31,6 +31,9 @@ public class NamespaceExtractionConfig @JsonProperty private int numExtractionThreads = 2; + @JsonProperty + private int numBufferedEntries = 100_000; + public int getNumExtractionThreads() { return numExtractionThreads; @@ -40,4 +43,16 @@ public class NamespaceExtractionConfig { this.numExtractionThreads = numExtractionThreads; } + + public int getNumBufferedEntries() + { + return numBufferedEntries; + } + + public void setNumBufferedEntries(int numBufferedEntries) + { + this.numBufferedEntries = numBufferedEntries; + } + + } diff --git a/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/OffHeapNamespaceExtractionCacheManager.java b/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/OffHeapNamespaceExtractionCacheManager.java index 149061b2003..0501974e21a 100644 --- a/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/OffHeapNamespaceExtractionCacheManager.java +++ b/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/OffHeapNamespaceExtractionCacheManager.java @@ -156,7 +156,7 @@ public class OffHeapNamespaceExtractionCacheManager extends NamespaceExtractionC .asyncWriteEnable() .mmapFileEnable() .commitFileSyncDisable() - .cacheSize(10_000_000) + .cacheSize(config.getNumBufferedEntries()) .make(); try { lifecycle.addMaybeStartHandler(