Fix filter cache setting to allow percentages
During adding the new settings infrastructure the option to specify the size of the filter cache as a percentage of the heap size which accidentally removed. This change adds that ability back. In addition the `Setting` class had multiple `.byteSizeSetting` methods which all except one used `ByteSizeValue.parseBytesSizeValue` to parse the value. One method used `MemorySizeValue.parseBytesSizeValueOrHeapRatio`. This was confusing as the way the value was parsed depended on how many arguments were provided. This change makes all `Setting.byteSizeSetting` methods parse the value the same way using `ByteSizeValue.parseBytesSizeValue` and adds `Setting.memorySizeSetting` methods to parse settings that express memory sizes (i.e. can be absolute bytes values or percentages). Relevant settings have been moved to use these new methods. Closes #20330
This commit is contained in:
Colin Goodheart-Smithe
parent
f825e8f4cb
commit
55d9e99f51
|
|
@ -551,10 +551,6 @@ public class Setting<T> extends ToXContentToBytes {
|
|||
return new Setting<>(key, defaultValueFn, Booleans::parseBooleanExact, properties);
|
||||
}
|
||||
|
||||
public static Setting<ByteSizeValue> byteSizeSetting(String key, String percentage, Property... properties) {
|
||||
return new Setting<>(key, (s) -> percentage, (s) -> MemorySizeValue.parseBytesSizeValueOrHeapRatio(s, key), properties);
|
||||
}
|
||||
|
||||
public static Setting<ByteSizeValue> byteSizeSetting(String key, ByteSizeValue value, Property... properties) {
|
||||
return byteSizeSetting(key, (s) -> value.toString(), properties);
|
||||
}
|
||||
|
|
@ -591,6 +587,49 @@ public class Setting<T> extends ToXContentToBytes {
|
|||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a setting which specifies a memory size. This can either be
|
||||
* specified as an absolute bytes value or as a percentage of the heap
|
||||
* memory.
|
||||
*
|
||||
* @param key the key for the setting
|
||||
* @param defaultValue the default value for this setting
|
||||
* @param properties properties properties for this setting like scope, filtering...
|
||||
* @return the setting object
|
||||
*/
|
||||
public static Setting<ByteSizeValue> memorySizeSetting(String key, ByteSizeValue defaultValue, Property... properties) {
|
||||
return memorySizeSetting(key, (s) -> defaultValue.toString(), properties);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Creates a setting which specifies a memory size. This can either be
|
||||
* specified as an absolute bytes value or as a percentage of the heap
|
||||
* memory.
|
||||
*
|
||||
* @param key the key for the setting
|
||||
* @param defaultValue a function that supplies the default value for this setting
|
||||
* @param properties properties properties for this setting like scope, filtering...
|
||||
* @return the setting object
|
||||
*/
|
||||
public static Setting<ByteSizeValue> memorySizeSetting(String key, Function<Settings, String> defaultValue, Property... properties) {
|
||||
return new Setting<>(key, defaultValue, (s) -> MemorySizeValue.parseBytesSizeValueOrHeapRatio(s, key), properties);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a setting which specifies a memory size. This can either be
|
||||
* specified as an absolute bytes value or as a percentage of the heap
|
||||
* memory.
|
||||
*
|
||||
* @param key the key for the setting
|
||||
* @param defaultPercentage the default value of this setting as a percentage of the heap memory
|
||||
* @param properties properties properties for this setting like scope, filtering...
|
||||
* @return the setting object
|
||||
*/
|
||||
public static Setting<ByteSizeValue> memorySizeSetting(String key, String defaultPercentage, Property... properties) {
|
||||
return new Setting<>(key, (s) -> defaultPercentage, (s) -> MemorySizeValue.parseBytesSizeValueOrHeapRatio(s, key), properties);
|
||||
}
|
||||
|
||||
public static Setting<TimeValue> positiveTimeSetting(String key, TimeValue defaultValue, Property... properties) {
|
||||
return timeSetting(key, defaultValue, TimeValue.timeValueMillis(0), properties);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ public class PageCacheRecycler extends AbstractComponent implements Releasable {
|
|||
public static final Setting<Type> TYPE_SETTING =
|
||||
new Setting<>("cache.recycler.page.type", Type.CONCURRENT.name(), Type::parse, Property.NodeScope);
|
||||
public static final Setting<ByteSizeValue> LIMIT_HEAP_SETTING =
|
||||
Setting.byteSizeSetting("cache.recycler.page.limit.heap", "10%", Property.NodeScope);
|
||||
Setting.memorySizeSetting("cache.recycler.page.limit.heap", "10%", Property.NodeScope);
|
||||
public static final Setting<Double> WEIGHT_BYTES_SETTING =
|
||||
Setting.doubleSetting("cache.recycler.page.weight.bytes", 1d, 0d, Property.NodeScope);
|
||||
public static final Setting<Double> WEIGHT_LONG_SETTING =
|
||||
|
|
|
|||
|
|
@ -52,7 +52,8 @@ import java.util.concurrent.locks.ReentrantLock;
|
|||
public class IndexingMemoryController extends AbstractComponent implements IndexingOperationListener, Closeable {
|
||||
|
||||
/** How much heap (% or bytes) we will share across all actively indexing shards on this node (default: 10%). */
|
||||
public static final Setting<ByteSizeValue> INDEX_BUFFER_SIZE_SETTING = Setting.byteSizeSetting("indices.memory.index_buffer_size", "10%", Property.NodeScope);
|
||||
public static final Setting<ByteSizeValue> INDEX_BUFFER_SIZE_SETTING =
|
||||
Setting.memorySizeSetting("indices.memory.index_buffer_size", "10%", Property.NodeScope);
|
||||
|
||||
/** Only applies when <code>indices.memory.index_buffer_size</code> is a %, to set a floor on the actual size in bytes (default: 48 MB). */
|
||||
public static final Setting<ByteSizeValue> MIN_INDEX_BUFFER_SIZE_SETTING = Setting.byteSizeSetting("indices.memory.min_index_buffer_size",
|
||||
|
|
|
|||
|
|
@ -49,13 +49,13 @@ import java.util.function.Predicate;
|
|||
|
||||
public class IndicesQueryCache extends AbstractComponent implements QueryCache, Closeable {
|
||||
|
||||
public static final Setting<ByteSizeValue> INDICES_CACHE_QUERY_SIZE_SETTING = Setting.byteSizeSetting(
|
||||
"indices.queries.cache.size", "10%", Property.NodeScope);
|
||||
public static final Setting<Integer> INDICES_CACHE_QUERY_COUNT_SETTING = Setting.intSetting(
|
||||
"indices.queries.cache.count", 10000, 1, Property.NodeScope);
|
||||
public static final Setting<ByteSizeValue> INDICES_CACHE_QUERY_SIZE_SETTING =
|
||||
Setting.memorySizeSetting("indices.queries.cache.size", "10%", Property.NodeScope);
|
||||
public static final Setting<Integer> INDICES_CACHE_QUERY_COUNT_SETTING =
|
||||
Setting.intSetting("indices.queries.cache.count", 10000, 1, Property.NodeScope);
|
||||
// enables caching on all segments instead of only the larger ones, for testing only
|
||||
public static final Setting<Boolean> INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING = Setting.boolSetting(
|
||||
"indices.queries.cache.all_segments", false, Property.NodeScope);
|
||||
public static final Setting<Boolean> INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING =
|
||||
Setting.boolSetting("indices.queries.cache.all_segments", false, Property.NodeScope);
|
||||
|
||||
private final LRUQueryCache cache;
|
||||
private final ShardCoreKeyMap shardKeyMap = new ShardCoreKeyMap();
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ public final class IndicesRequestCache extends AbstractComponent implements Remo
|
|||
public static final Setting<Boolean> INDEX_CACHE_REQUEST_ENABLED_SETTING =
|
||||
Setting.boolSetting("index.requests.cache.enable", true, Property.Dynamic, Property.IndexScope);
|
||||
public static final Setting<ByteSizeValue> INDICES_CACHE_QUERY_SIZE =
|
||||
Setting.byteSizeSetting("indices.requests.cache.size", "1%", Property.NodeScope);
|
||||
Setting.memorySizeSetting("indices.requests.cache.size", "1%", Property.NodeScope);
|
||||
public static final Setting<TimeValue> INDICES_CACHE_QUERY_EXPIRE =
|
||||
Setting.positiveTimeSetting("indices.requests.cache.expire", new TimeValue(0), Property.NodeScope);
|
||||
|
||||
|
|
|
|||
|
|
@ -47,24 +47,24 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
|
|||
private final ConcurrentMap<String, CircuitBreaker> breakers = new ConcurrentHashMap<>();
|
||||
|
||||
public static final Setting<ByteSizeValue> TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING =
|
||||
Setting.byteSizeSetting("indices.breaker.total.limit", "70%", Property.Dynamic, Property.NodeScope);
|
||||
Setting.memorySizeSetting("indices.breaker.total.limit", "70%", Property.Dynamic, Property.NodeScope);
|
||||
|
||||
public static final Setting<ByteSizeValue> FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING =
|
||||
Setting.byteSizeSetting("indices.breaker.fielddata.limit", "60%", Property.Dynamic, Property.NodeScope);
|
||||
Setting.memorySizeSetting("indices.breaker.fielddata.limit", "60%", Property.Dynamic, Property.NodeScope);
|
||||
public static final Setting<Double> FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING =
|
||||
Setting.doubleSetting("indices.breaker.fielddata.overhead", 1.03d, 0.0d, Property.Dynamic, Property.NodeScope);
|
||||
public static final Setting<CircuitBreaker.Type> FIELDDATA_CIRCUIT_BREAKER_TYPE_SETTING =
|
||||
new Setting<>("indices.breaker.fielddata.type", "memory", CircuitBreaker.Type::parseValue, Property.NodeScope);
|
||||
|
||||
public static final Setting<ByteSizeValue> REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING =
|
||||
Setting.byteSizeSetting("indices.breaker.request.limit", "60%", Property.Dynamic, Property.NodeScope);
|
||||
Setting.memorySizeSetting("indices.breaker.request.limit", "60%", Property.Dynamic, Property.NodeScope);
|
||||
public static final Setting<Double> REQUEST_CIRCUIT_BREAKER_OVERHEAD_SETTING =
|
||||
Setting.doubleSetting("indices.breaker.request.overhead", 1.0d, 0.0d, Property.Dynamic, Property.NodeScope);
|
||||
public static final Setting<CircuitBreaker.Type> REQUEST_CIRCUIT_BREAKER_TYPE_SETTING =
|
||||
new Setting<>("indices.breaker.request.type", "memory", CircuitBreaker.Type::parseValue, Property.NodeScope);
|
||||
|
||||
public static final Setting<ByteSizeValue> IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING =
|
||||
Setting.byteSizeSetting("network.breaker.inflight_requests.limit", "100%", Property.Dynamic, Property.NodeScope);
|
||||
Setting.memorySizeSetting("network.breaker.inflight_requests.limit", "100%", Property.Dynamic, Property.NodeScope);
|
||||
public static final Setting<Double> IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_OVERHEAD_SETTING =
|
||||
Setting.doubleSetting("network.breaker.inflight_requests.overhead", 1.0d, 0.0d, Property.Dynamic, Property.NodeScope);
|
||||
public static final Setting<CircuitBreaker.Type> IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_TYPE_SETTING =
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ import java.util.function.ToLongBiFunction;
|
|||
public class IndicesFieldDataCache extends AbstractComponent implements RemovalListener<IndicesFieldDataCache.Key, Accountable>, Releasable{
|
||||
|
||||
public static final Setting<ByteSizeValue> INDICES_FIELDDATA_CACHE_SIZE_KEY =
|
||||
Setting.byteSizeSetting("indices.fielddata.cache.size", new ByteSizeValue(-1), Property.NodeScope);
|
||||
Setting.memorySizeSetting("indices.fielddata.cache.size", new ByteSizeValue(-1), Property.NodeScope);
|
||||
private final IndexFieldDataCache.Listener indicesFieldDataCacheListener;
|
||||
private final Cache<Key, Accountable> cache;
|
||||
|
||||
|
|
|
|||
|
|
@ -54,9 +54,9 @@ public class FsRepository extends BlobStoreRepository {
|
|||
public static final Setting<String> REPOSITORIES_LOCATION_SETTING =
|
||||
new Setting<>("repositories.fs.location", LOCATION_SETTING, Function.identity(), Property.NodeScope);
|
||||
public static final Setting<ByteSizeValue> CHUNK_SIZE_SETTING =
|
||||
Setting.byteSizeSetting("chunk_size", "-1", Property.NodeScope);
|
||||
Setting.byteSizeSetting("chunk_size", new ByteSizeValue(-1), Property.NodeScope);
|
||||
public static final Setting<ByteSizeValue> REPOSITORIES_CHUNK_SIZE_SETTING =
|
||||
Setting.byteSizeSetting("repositories.fs.chunk_size", "-1", Property.NodeScope);
|
||||
Setting.byteSizeSetting("repositories.fs.chunk_size", new ByteSizeValue(-1), Property.NodeScope);
|
||||
public static final Setting<Boolean> COMPRESS_SETTING = Setting.boolSetting("compress", false, Property.NodeScope);
|
||||
public static final Setting<Boolean> REPOSITORIES_COMPRESS_SETTING =
|
||||
Setting.boolSetting("repositories.fs.compress", false, Property.NodeScope);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,88 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.common.settings;
|
||||
|
||||
import org.elasticsearch.common.settings.Setting.Property;
|
||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||
import org.elasticsearch.common.util.PageCacheRecycler;
|
||||
import org.elasticsearch.indices.IndexingMemoryController;
|
||||
import org.elasticsearch.indices.IndicesQueryCache;
|
||||
import org.elasticsearch.indices.IndicesRequestCache;
|
||||
import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
|
||||
import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
|
||||
import org.elasticsearch.monitor.jvm.JvmInfo;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
|
||||
import static org.hamcrest.Matchers.notNullValue;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.hasItem;
|
||||
|
||||
public class MemorySizeSettingsTests extends ESTestCase {
|
||||
|
||||
public void testPageCacheLimitHeapSetting() {
|
||||
assertMemorySizeSetting(PageCacheRecycler.LIMIT_HEAP_SETTING, "cache.recycler.page.limit.heap",
|
||||
new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.1)));
|
||||
}
|
||||
|
||||
public void testIndexBufferSizeSetting() {
|
||||
assertMemorySizeSetting(IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING, "indices.memory.index_buffer_size",
|
||||
new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.1)));
|
||||
}
|
||||
|
||||
public void testQueryCacheSizeSetting() {
|
||||
assertMemorySizeSetting(IndicesQueryCache.INDICES_CACHE_QUERY_SIZE_SETTING, "indices.queries.cache.size",
|
||||
new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.1)));
|
||||
}
|
||||
|
||||
public void testIndicesRequestCacheSetting() {
|
||||
assertMemorySizeSetting(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE, "indices.requests.cache.size",
|
||||
new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.01)));
|
||||
}
|
||||
|
||||
public void testCircuitBreakerSettings() {
|
||||
assertMemorySizeSetting(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING, "indices.breaker.total.limit",
|
||||
new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.7)));
|
||||
assertMemorySizeSetting(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING, "indices.breaker.fielddata.limit",
|
||||
new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.6)));
|
||||
assertMemorySizeSetting(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING, "indices.breaker.request.limit",
|
||||
new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.6)));
|
||||
assertMemorySizeSetting(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING,
|
||||
"network.breaker.inflight_requests.limit", new ByteSizeValue((JvmInfo.jvmInfo().getMem().getHeapMax().bytes())));
|
||||
}
|
||||
|
||||
public void testIndicesFieldDataCacheSetting() {
|
||||
assertMemorySizeSetting(IndicesFieldDataCache.INDICES_FIELDDATA_CACHE_SIZE_KEY, "indices.fielddata.cache.size",
|
||||
new ByteSizeValue(-1));
|
||||
}
|
||||
|
||||
private void assertMemorySizeSetting(Setting<ByteSizeValue> setting, String settingKey, ByteSizeValue defaultValue) {
|
||||
assertThat(setting, notNullValue());
|
||||
assertThat(setting.getKey(), equalTo(settingKey));
|
||||
assertThat(setting.getProperties(), hasItem(Property.NodeScope));
|
||||
assertThat(setting.getDefault(Settings.EMPTY),
|
||||
equalTo(defaultValue));
|
||||
Settings settingWithPercentage = Settings.builder().put(settingKey, "25%").build();
|
||||
assertThat(setting.get(settingWithPercentage),
|
||||
equalTo(new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.25))));
|
||||
Settings settingWithBytesValue = Settings.builder().put(settingKey, "1024b").build();
|
||||
assertThat(setting.get(settingWithBytesValue), equalTo(new ByteSizeValue(1024)));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -22,6 +22,7 @@ import org.elasticsearch.common.collect.Tuple;
|
|||
import org.elasticsearch.common.settings.Setting.Property;
|
||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.monitor.jvm.JvmInfo;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
|
@ -68,6 +69,44 @@ public class SettingTests extends ESTestCase {
|
|||
assertEquals(new ByteSizeValue(12), value.get());
|
||||
}
|
||||
|
||||
public void testMemorySize() {
|
||||
Setting<ByteSizeValue> memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", new ByteSizeValue(1024), Property.Dynamic,
|
||||
Property.NodeScope);
|
||||
|
||||
assertFalse(memorySizeValueSetting.isGroupSetting());
|
||||
ByteSizeValue memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY);
|
||||
assertEquals(memorySizeValue.bytes(), 1024);
|
||||
|
||||
memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", s -> "2048b", Property.Dynamic, Property.NodeScope);
|
||||
memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY);
|
||||
assertEquals(memorySizeValue.bytes(), 2048);
|
||||
|
||||
memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", "50%", Property.Dynamic, Property.NodeScope);
|
||||
assertFalse(memorySizeValueSetting.isGroupSetting());
|
||||
memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY);
|
||||
assertEquals(memorySizeValue.bytes(), JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.5, 1.0);
|
||||
|
||||
memorySizeValueSetting = Setting.memorySizeSetting("a.byte.size", s -> "25%", Property.Dynamic, Property.NodeScope);
|
||||
memorySizeValue = memorySizeValueSetting.get(Settings.EMPTY);
|
||||
assertEquals(memorySizeValue.bytes(), JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.25, 1.0);
|
||||
|
||||
AtomicReference<ByteSizeValue> value = new AtomicReference<>(null);
|
||||
ClusterSettings.SettingUpdater<ByteSizeValue> settingUpdater = memorySizeValueSetting.newUpdater(value::set, logger);
|
||||
try {
|
||||
settingUpdater.apply(Settings.builder().put("a.byte.size", 12).build(), Settings.EMPTY);
|
||||
fail("no unit");
|
||||
} catch (IllegalArgumentException ex) {
|
||||
assertEquals("failed to parse setting [a.byte.size] with value [12] as a size in bytes: unit is missing or unrecognized",
|
||||
ex.getMessage());
|
||||
}
|
||||
|
||||
assertTrue(settingUpdater.apply(Settings.builder().put("a.byte.size", "12b").build(), Settings.EMPTY));
|
||||
assertEquals(new ByteSizeValue(12), value.get());
|
||||
|
||||
assertTrue(settingUpdater.apply(Settings.builder().put("a.byte.size", "20%").build(), Settings.EMPTY));
|
||||
assertEquals(new ByteSizeValue((int) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.2)), value.get());
|
||||
}
|
||||
|
||||
public void testSimpleUpdate() {
|
||||
Setting<Boolean> booleanSetting = Setting.boolSetting("foo.bar", false, Property.Dynamic, Property.NodeScope);
|
||||
AtomicReference<Boolean> atomicBoolean = new AtomicReference<>(null);
|
||||
|
|
|
|||
Loading…
Reference in New Issue