honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow affix settings to specify dependencies (#27161) 

We use affix settings to group settings / values under a certain namespace.
In some cases like login information for instance a setting is only valid if
one or more other settings are present. For instance `x.test.user` is only valid
if there is an `x.test.passwd` present and vice versa. This change allows to specify
such a dependency to prevent settings updates that leave settings in an inconsistent
state.
This commit is contained in:
Simon Willnauer 2017-11-13 12:06:36 +01:00 committed by GitHub
parent 08037eebff
commit 2299c70371
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 309 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
core/src/main/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdater.java
View File

@ -54,15 +54,23 @@ final class SettingsUpdater {
transientSettings.put(currentState.metaData().transientSettings());
changed |= clusterSettings.updateDynamicSettings(transientToApply, transientSettings, transientUpdates, "transient");
Settings.Builder persistentSettings = Settings.builder();
persistentSettings.put(currentState.metaData().persistentSettings());
changed |= clusterSettings.updateDynamicSettings(persistentToApply, persistentSettings, persistentUpdates, "persistent");
final ClusterState clusterState;
if (changed) {
Settings transientFinalSettings = transientSettings.build();
Settings persistentFinalSettings = persistentSettings.build();
// both transient and persistent settings must be consistent by itself we can't allow dependencies to be
// in either of them otherwise a full cluster restart will break the settings validation
clusterSettings.validate(transientFinalSettings, true);
clusterSettings.validate(persistentFinalSettings, true);
MetaData.Builder metaData = MetaData.builder(currentState.metaData())
.persistentSettings(persistentSettings.build())
.transientSettings(transientSettings.build());
.persistentSettings(persistentFinalSettings)
.transientSettings(transientFinalSettings);
ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
boolean updatedReadOnly = MetaData.SETTING_READ_ONLY_SETTING.get(metaData.persistentSettings())

2
core/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java
View File

@ -77,7 +77,7 @@ public class TransportPutIndexTemplateAction extends TransportMasterNodeAction<P
}
final Settings.Builder templateSettingsBuilder = Settings.builder();
templateSettingsBuilder.put(request.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
indexScopedSettings.validate(templateSettingsBuilder);
indexScopedSettings.validate(templateSettingsBuilder.build(), true); // templates must be consistent with regards to dependencies
indexTemplateService.putTemplate(new MetaDataIndexTemplateService.PutRequest(cause, request.name())
.patterns(request.patterns())
.order(request.order())

8
core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View File

@ -220,10 +220,9 @@ public class MetaDataCreateIndexService extends AbstractComponent {
private void onlyCreateIndex(final CreateIndexClusterStateUpdateRequest request,
final ActionListener<ClusterStateUpdateResponse> listener) {
Settings.Builder updatedSettingsBuilder = Settings.builder();
updatedSettingsBuilder.put(request.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
indexScopedSettings.validate(updatedSettingsBuilder);
request.settings(updatedSettingsBuilder.build());
Settings build = updatedSettingsBuilder.put(request.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build();
indexScopedSettings.validate(build, true); // we do validate here - index setting must be consistent
request.settings(build);
clusterService.submitStateUpdateTask("create-index [" + request.index() + "], cause [" + request.cause() + "]",
new IndexCreationTask(logger, allocationService, request, listener, indicesService, aliasValidator, xContentRegistry, settings,
this::validate));
@ -420,7 +419,6 @@ public class MetaDataCreateIndexService extends AbstractComponent {
tmpImdBuilder.primaryTerm(shardId, primaryTerm);
}
}
// Set up everything, now locally create the index to see that things are ok, and apply
final IndexMetaData tmpImd = tmpImdBuilder.build();
ActiveShardCount waitForActiveShards = request.waitForActiveShards();

2
core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
View File

@ -276,7 +276,7 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
}
try {
indexScopedSettings.validate(request.settings);
indexScopedSettings.validate(request.settings, true); // templates must be consistent with regards to dependencies
} catch (IllegalArgumentException iae) {
validationErrors.add(iae.getMessage());
for (Throwable t : iae.getSuppressed()) {

11
core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
View File

@ -54,6 +54,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.function.Predicate;
import static org.elasticsearch.action.support.ContextPreservingActionListener.wrapPreservingContext;
@ -163,7 +164,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
Settings.Builder settingsForOpenIndices = Settings.builder();
final Set<String> skippedSettings = new HashSet<>();
indexScopedSettings.validate(normalizedSettings);
indexScopedSettings.validate(normalizedSettings, false); // don't validate dependencies here we check it below
// never allow to change the number of shards
for (String key : normalizedSettings.keySet()) {
Setting setting = indexScopedSettings.get(key);
@ -240,7 +241,9 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
if (preserveExisting) {
indexSettings.put(indexMetaData.getSettings());
}
metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(indexSettings));
Settings finalSettings = indexSettings.build();
indexScopedSettings.validate(finalSettings.filter(k -> indexScopedSettings.isPrivateSetting(k) == false), true);
metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(finalSettings));
}
}
}
@ -254,7 +257,9 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
if (preserveExisting) {
indexSettings.put(indexMetaData.getSettings());
}
metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(indexSettings));
Settings finalSettings = indexSettings.build();
indexScopedSettings.validate(finalSettings.filter(k -> indexScopedSettings.isPrivateSetting(k) == false), true);
metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(finalSettings));
}
}
}

55
core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
View File

@ -264,20 +264,16 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
}
/**
* Validates that all settings in the builder are registered and valid
* Validates that all given settings are registered and valid
* @param settings the settings to validate
* @param validateDependencies if <code>true</code> settings dependencies are validated as well.
* @see Setting#getSettingsDependencies(String)
*/
public final void validate(Settings.Builder settingsBuilder) {
validate(settingsBuilder.build());
}
/**
* * Validates that all given settings are registered and valid
*/
public final void validate(Settings settings) {
public final void validate(Settings settings, boolean validateDependencies) {
List<RuntimeException> exceptions = new ArrayList<>();
for (String key : settings.keySet()) { // settings iterate in deterministic fashion
try {
validate(key, settings);
validate(key, settings, validateDependencies);
} catch (RuntimeException ex) {
exceptions.add(ex);
}
@ -285,12 +281,11 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
ExceptionsHelper.rethrowAndSuppress(exceptions);
}
/**
* Validates that the setting is valid
*/
public final void validate(String key, Settings settings) {
Setting setting = get(key);
void validate(String key, Settings settings, boolean validateDependencies) {
Setting setting = getRaw(key);
if (setting == null) {
LevensteinDistance ld = new LevensteinDistance();
List<Tuple<Float, String>> scoredKeys = new ArrayList<>();
@ -315,6 +310,20 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
"settings";
}
throw new IllegalArgumentException(msg);
} else {
Set<String> settingsDependencies = setting.getSettingsDependencies(key);
if (setting.hasComplexMatcher()) {
setting = setting.getConcreteSetting(key);
}
if (validateDependencies && settingsDependencies.isEmpty() == false) {
Set<String> settingKeys = settings.keySet();
for (String requiredSetting : settingsDependencies) {
if (settingKeys.contains(requiredSetting) == false) {
throw new IllegalArgumentException("Missing required setting ["
+ requiredSetting + "] for setting [" + setting.getKey() + "]");
}
}
}
}
setting.get(settings);
}
@ -375,7 +384,18 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
/**
* Returns the {@link Setting} for the given key or <code>null</code> if the setting can not be found.
*/
public Setting<?> get(String key) {
public final Setting<?> get(String key) {
Setting<?> raw = getRaw(key);
if (raw == null) {
return null;
} if (raw.hasComplexMatcher()) {
return raw.getConcreteSetting(key);
} else {
return raw;
}
}
private Setting<?> getRaw(String key) {
Setting<?> setting = keySettings.get(key);
if (setting != null) {
return setting;
@ -383,7 +403,8 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
for (Map.Entry<String, Setting<?>> entry : complexMatchers.entrySet()) {
if (entry.getValue().match(key)) {
assert assertMatcher(key, 1);
return entry.getValue().getConcreteSetting(key);
assert entry.getValue().hasComplexMatcher();
return entry.getValue();
}
}
return null;
@ -513,7 +534,7 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
} else if (get(key) == null) {
throw new IllegalArgumentException(type + " setting [" + key + "], not recognized");
} else if (isNull == false && canUpdate.test(key)) {
validate(key, toApply);
validate(key, toApply, false); // we might not have a full picture here do to a dependency validation
settingsBuilder.copy(key, toApply);
updates.copy(key, toApply);
changed = true;
@ -654,7 +675,7 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
* representation. Otherwise <code>false</code>
*/
// TODO this should be replaced by Setting.Property.HIDDEN or something like this.
protected boolean isPrivateSetting(String key) {
public boolean isPrivateSetting(String key) {
return false;
}
}

2
core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
View File

@ -191,7 +191,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
}
@Override
protected boolean isPrivateSetting(String key) {
public boolean isPrivateSetting(String key) {
switch (key) {
case IndexMetaData.SETTING_CREATION_DATE:
case IndexMetaData.SETTING_INDEX_UUID:

41
core/src/main/java/org/elasticsearch/common/settings/Setting.java
View File

@ -42,6 +42,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.List;
@ -126,7 +127,7 @@ public class Setting<T> implements ToXContentObject {
private static final EnumSet<Property> EMPTY_PROPERTIES = EnumSet.noneOf(Property.class);
private Setting(Key key, @Nullable Setting<T> fallbackSetting, Function<Settings, String> defaultValue, Function<String, T> parser,
Validator<T> validator, Property... properties) {
Validator<T> validator, Property... properties) {
assert this instanceof SecureSetting || this.isGroupSetting() || parser.apply(defaultValue.apply(Settings.EMPTY)) != null
: "parser returned null";
this.key = key;
@ -457,6 +458,14 @@ public class Setting<T> implements ToXContentObject {
return this;
}
/**
* Returns a set of settings that are required at validation time. Unless all of the dependencies are present in the settings
* object validation of setting must fail.
*/
public Set<String> getSettingsDependencies(String key) {
return Collections.emptySet();
}
/**
* Build a new updater with a noop validator.
*/
@ -519,11 +528,13 @@ public class Setting<T> implements ToXContentObject {
public static class AffixSetting<T> extends Setting<T> {
private final AffixKey key;
private final Function<String, Setting<T>> delegateFactory;
private final Set<AffixSetting> dependencies;
public AffixSetting(AffixKey key, Setting<T> delegate, Function<String, Setting<T>> delegateFactory) {
public AffixSetting(AffixKey key, Setting<T> delegate, Function<String, Setting<T>> delegateFactory, AffixSetting... dependencies) {
super(key, delegate.defaultValue, delegate.parser, delegate.properties.toArray(new Property[0]));
this.key = key;
this.delegateFactory = delegateFactory;
this.dependencies = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(dependencies)));
}
boolean isGroupSetting() {
@ -534,6 +545,15 @@ public class Setting<T> implements ToXContentObject {
return settings.keySet().stream().filter((key) -> match(key)).map(settingKey -> key.getConcreteString(settingKey));
}
public Set<String> getSettingsDependencies(String settingsKey) {
if (dependencies.isEmpty()) {
return Collections.emptySet();
} else {
String namespace = key.getNamespace(settingsKey);
return dependencies.stream().map(s -> s.key.toConcreteKey(namespace).key).collect(Collectors.toSet());
}
}
AbstractScopedSettings.SettingUpdater<Map<AbstractScopedSettings.SettingUpdater<T>, T>> newAffixUpdater(
BiConsumer<String, T> consumer, Logger logger, BiConsumer<String, T> validator) {
return new AbstractScopedSettings.SettingUpdater<Map<AbstractScopedSettings.SettingUpdater<T>, T>>() {
@ -659,6 +679,13 @@ public class Setting<T> implements ToXContentObject {
return matchStream(settings).distinct().map(this::getConcreteSetting);
}
/**
* Returns distinct namespaces for the given settings
*/
public Set<String> getNamespaces(Settings settings) {
return settings.keySet().stream().filter(this::match).map(key::getNamespace).collect(Collectors.toSet());
}
/**
* Returns a map of all namespaces to it's values give the provided settings
*/
@ -1184,13 +1211,15 @@ public class Setting<T> implements ToXContentObject {
* storage.${backend}.enable=[true|false] can easily be added with this setting. Yet, affix key settings don't support updaters
* out of the box unless {@link #getConcreteSetting(String)} is used to pull the updater.
*/
public static <T> AffixSetting<T> affixKeySetting(String prefix, String suffix, Function<String, Setting<T>> delegateFactory) {
return affixKeySetting(new AffixKey(prefix, suffix), delegateFactory);
public static <T> AffixSetting<T> affixKeySetting(String prefix, String suffix, Function<String, Setting<T>> delegateFactory,
AffixSetting... dependencies) {
return affixKeySetting(new AffixKey(prefix, suffix), delegateFactory, dependencies);
}
private static <T> AffixSetting<T> affixKeySetting(AffixKey key, Function<String, Setting<T>> delegateFactory) {
private static <T> AffixSetting<T> affixKeySetting(AffixKey key, Function<String, Setting<T>> delegateFactory,
AffixSetting... dependencies) {
Setting<T> delegate = delegateFactory.apply("_na_");
return new AffixSetting<>(key, delegate, delegateFactory);
return new AffixSetting<>(key, delegate, delegateFactory, dependencies);
};

2
core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
View File

@ -132,7 +132,7 @@ public class SettingsModule implements Module {
}
}
// by now we are fully configured, lets check node level settings for unregistered index settings
clusterSettings.validate(settings);
clusterSettings.validate(settings, true);
this.settingsFilter = new SettingsFilter(settings, settingsFilterPattern);
}

47
core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
View File

@ -122,7 +122,7 @@ public class ScopedSettingsTests extends ESTestCase {
Settings.Builder builder = Settings.builder();
Settings updates = Settings.builder().putNull("index.routing.allocation.require._ip")
.put("index.some.dyn.setting", 1).build();
settings.validate(updates);
settings.validate(updates, false);
settings.updateDynamicSettings(updates,
Settings.builder().put(currentSettings), builder, "node");
currentSettings = builder.build();
@ -160,6 +160,26 @@ public class ScopedSettingsTests extends ESTestCase {
assertEquals(0, consumer2.get());
}
public void testDependentSettings() {
Setting.AffixSetting<String> stringSetting = Setting.affixKeySetting("foo.", "name",
(k) -> Setting.simpleString(k, Property.Dynamic, Property.NodeScope));
Setting.AffixSetting<Integer> intSetting = Setting.affixKeySetting("foo.", "bar",
(k) -> Setting.intSetting(k, 1, Property.Dynamic, Property.NodeScope), stringSetting);
AbstractScopedSettings service = new ClusterSettings(Settings.EMPTY,new HashSet<>(Arrays.asList(intSetting, stringSetting)));
IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
() -> service.validate(Settings.builder().put("foo.test.bar", 7).build(), true));
assertEquals("Missing required setting [foo.test.name] for setting [foo.test.bar]", iae.getMessage());
service.validate(Settings.builder()
.put("foo.test.name", "test")
.put("foo.test.bar", 7)
.build(), true);
service.validate(Settings.builder().put("foo.test.bar", 7).build(), false);
}
public void testAddConsumerAffix() {
Setting.AffixSetting<Integer> intSetting = Setting.affixKeySetting("foo.", "bar",
(k) -> Setting.intSetting(k, 1, Property.Dynamic, Property.NodeScope));
@ -585,7 +605,7 @@ public class ScopedSettingsTests extends ESTestCase {
Settings.EMPTY,
IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
() -> settings.validate(Settings.builder().put("index.numbe_of_replica", "1").build()));
() -> settings.validate(Settings.builder().put("index.numbe_of_replica", "1").build(), false));
assertEquals(iae.getMessage(), "unknown setting [index.numbe_of_replica] did you mean [index.number_of_replicas]?");
}
@ -595,26 +615,23 @@ public class ScopedSettingsTests extends ESTestCase {
IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
String unknownMsgSuffix = " please check that any required plugins are installed, or check the breaking changes documentation for" +
" removed settings";
settings.validate(Settings.builder().put("index.store.type", "boom"));
settings.validate(Settings.builder().put("index.store.type", "boom").build());
settings.validate(Settings.builder().put("index.store.type", "boom").build(), false);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
settings.validate(Settings.builder().put("index.store.type", "boom").put("i.am.not.a.setting", true)));
settings.validate(Settings.builder().put("index.store.type", "boom").put("i.am.not.a.setting", true).build(), false));
assertEquals("unknown setting [i.am.not.a.setting]" + unknownMsgSuffix, e.getMessage());
e = expectThrows(IllegalArgumentException.class, () ->
settings.validate(Settings.builder().put("index.store.type", "boom").put("i.am.not.a.setting", true).build()));
assertEquals("unknown setting [i.am.not.a.setting]" + unknownMsgSuffix, e.getMessage());
e = expectThrows(IllegalArgumentException.class, () ->
settings.validate(Settings.builder().put("index.store.type", "boom").put("index.number_of_replicas", true).build()));
settings.validate(Settings.builder().put("index.store.type", "boom").put("index.number_of_replicas", true).build(), false));
assertEquals("Failed to parse value [true] for setting [index.number_of_replicas]", e.getMessage());
e = expectThrows(IllegalArgumentException.class, () ->
settings.validate("index.number_of_replicas", Settings.builder().put("index.number_of_replicas", "true").build()));
settings.validate("index.number_of_replicas", Settings.builder().put("index.number_of_replicas", "true").build(), false));
assertEquals("Failed to parse value [true] for setting [index.number_of_replicas]", e.getMessage());
e = expectThrows(IllegalArgumentException.class, () ->
settings.validate("index.similarity.classic.type", Settings.builder().put("index.similarity.classic.type", "mine").build()));
settings.validate("index.similarity.classic.type", Settings.builder().put("index.similarity.classic.type", "mine").build(),
false));
assertEquals("illegal value for [index.similarity.classic] cannot redefine built-in similarity", e.getMessage());
}
@ -624,12 +641,12 @@ public class ScopedSettingsTests extends ESTestCase {
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
final ClusterSettings clusterSettings = new ClusterSettings(settings, Collections.emptySet());
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.validate(settings));
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.validate(settings, false));
assertThat(e.getMessage(), startsWith("unknown secure setting [some.secure.setting]"));
ClusterSettings clusterSettings2 = new ClusterSettings(settings,
Collections.singleton(SecureSetting.secureString("some.secure.setting", null)));
clusterSettings2.validate(settings);
clusterSettings2.validate(settings, false);
}
public void testDiffSecureSettings() {
@ -722,7 +739,7 @@ public class ScopedSettingsTests extends ESTestCase {
IllegalArgumentException ex =
expectThrows(
IllegalArgumentException.class,
() -> settings.validate(Settings.builder().put("logger._root", "boom").build()));
() -> settings.validate(Settings.builder().put("logger._root", "boom").build(), false));
assertEquals("Unknown level constant [BOOM].", ex.getMessage());
assertEquals(level, ESLoggerFactory.getRootLogger().getLevel());
settings.applySettings(Settings.builder().put("logger._root", "TRACE").build());

18
core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
View File

@ -30,6 +30,7 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -42,6 +43,7 @@ import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
public class SettingTests extends ESTestCase {
public void testGet() {
Setting<Boolean> booleanSetting = Setting.boolSetting("foo.bar", false, Property.Dynamic, Property.NodeScope);
assertFalse(booleanSetting.get(Settings.EMPTY));
@ -577,6 +579,22 @@ public class SettingTests extends ESTestCase {
assertFalse(listAffixSetting.match("foo"));
}
public void testAffixSettingNamespaces() {
Setting.AffixSetting<Boolean> setting =
Setting.affixKeySetting("foo.", "enable", (key) -> Setting.boolSetting(key, false, Property.NodeScope));
Settings build = Settings.builder()
.put("foo.bar.enable", "true")
.put("foo.baz.enable", "true")
.put("foo.boom.enable", "true")
.put("something.else", "true")
.build();
Set<String> namespaces = setting.getNamespaces(build);
assertEquals(3, namespaces.size());
assertTrue(namespaces.contains("bar"));
assertTrue(namespaces.contains("baz"));
assertTrue(namespaces.contains("boom"));
}
public void testAffixAsMap() {
Setting.AffixSetting<String> setting = Setting.prefixKeySetting("foo.bar.", key ->
Setting.simpleString(key, Property.NodeScope));

2
core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
View File

@ -498,7 +498,7 @@ public class IndexSettingsTests extends ESTestCase {
assertTrue(index.isSingleType());
expectThrows(IllegalArgumentException.class, () -> {
index.getScopedSettings()
.validate(Settings.builder().put(IndexSettings.INDEX_MAPPING_SINGLE_TYPE_SETTING_KEY, randomBoolean()).build());
.validate(Settings.builder().put(IndexSettings.INDEX_MAPPING_SINGLE_TYPE_SETTING_KEY, randomBoolean()).build(), false);
});
}
{

132
core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
View File

@ -85,6 +85,17 @@ public class UpdateSettingsIT extends ESIntegTestCase {
public static class DummySettingPlugin extends Plugin {
public static final Setting<String> DUMMY_SETTING = Setting.simpleString("index.dummy",
Setting.Property.IndexScope, Setting.Property.Dynamic);
public static final Setting.AffixSetting<String> DUMMY_ACCOUNT_USER = Setting.affixKeySetting("index.acc.", "user",
k -> Setting.simpleString(k, Setting.Property.IndexScope, Setting.Property.Dynamic));
public static final Setting<String> DUMMY_ACCOUNT_PW = Setting.affixKeySetting("index.acc.", "pw",
k -> Setting.simpleString(k, Setting.Property.IndexScope, Setting.Property.Dynamic), DUMMY_ACCOUNT_USER);
public static final Setting.AffixSetting<String> DUMMY_ACCOUNT_USER_CLUSTER = Setting.affixKeySetting("cluster.acc.", "user",
k -> Setting.simpleString(k, Setting.Property.NodeScope, Setting.Property.Dynamic));
public static final Setting<String> DUMMY_ACCOUNT_PW_CLUSTER = Setting.affixKeySetting("cluster.acc.", "pw",
k -> Setting.simpleString(k, Setting.Property.NodeScope, Setting.Property.Dynamic), DUMMY_ACCOUNT_USER_CLUSTER);
@Override
public void onIndexModule(IndexModule indexModule) {
indexModule.addSettingsUpdateConsumer(DUMMY_SETTING, (s) -> {}, (s) -> {
@ -95,7 +106,8 @@ public class UpdateSettingsIT extends ESIntegTestCase {
@Override
public List<Setting<?>> getSettings() {
return Collections.singletonList(DUMMY_SETTING);
return Arrays.asList(DUMMY_SETTING, DUMMY_ACCOUNT_PW, DUMMY_ACCOUNT_USER,
DUMMY_ACCOUNT_PW_CLUSTER, DUMMY_ACCOUNT_USER_CLUSTER);
}
}
@ -112,6 +124,124 @@ public class UpdateSettingsIT extends ESIntegTestCase {
}
}
public void testUpdateDependentClusterSettings() {
IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder()
.put("cluster.acc.test.pw", "asdf")).get());
assertEquals("Missing required setting [cluster.acc.test.user] for setting [cluster.acc.test.pw]", iae.getMessage());
iae = expectThrows(IllegalArgumentException.class, () ->
client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
.put("cluster.acc.test.pw", "asdf")).get());
assertEquals("Missing required setting [cluster.acc.test.user] for setting [cluster.acc.test.pw]", iae.getMessage());
iae = expectThrows(IllegalArgumentException.class, () ->
client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
.put("cluster.acc.test.pw", "asdf")).setPersistentSettings(Settings.builder()
.put("cluster.acc.test.user", "asdf")).get());
assertEquals("Missing required setting [cluster.acc.test.user] for setting [cluster.acc.test.pw]", iae.getMessage());
if (randomBoolean()) {
client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
.put("cluster.acc.test.pw", "asdf")
.put("cluster.acc.test.user", "asdf")).get();
iae = expectThrows(IllegalArgumentException.class, () ->
client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
.putNull("cluster.acc.test.user")).get());
assertEquals("Missing required setting [cluster.acc.test.user] for setting [cluster.acc.test.pw]", iae.getMessage());
client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
.putNull("cluster.acc.test.pw")
.putNull("cluster.acc.test.user")).get();
} else {
client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder()
.put("cluster.acc.test.pw", "asdf")
.put("cluster.acc.test.user", "asdf")).get();
iae = expectThrows(IllegalArgumentException.class, () ->
client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder()
.putNull("cluster.acc.test.user")).get());
assertEquals("Missing required setting [cluster.acc.test.user] for setting [cluster.acc.test.pw]", iae.getMessage());
client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder()
.putNull("cluster.acc.test.pw")
.putNull("cluster.acc.test.user")).get();
}
}
public void testUpdateDependentIndexSettings() {
IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
prepareCreate("test", Settings.builder().put("index.acc.test.pw", "asdf")).get());
assertEquals("Missing required setting [index.acc.test.user] for setting [index.acc.test.pw]", iae.getMessage());
createIndex("test");
for (int i = 0; i < 2; i++) {
if (i == 1) {
// now do it on a closed index
client().admin().indices().prepareClose("test").get();
}
iae = expectThrows(IllegalArgumentException.class, () ->
client()
.admin()
.indices()
.prepareUpdateSettings("test")
.setSettings(
Settings.builder()
.put("index.acc.test.pw", "asdf"))
.execute()
.actionGet());
assertEquals("Missing required setting [index.acc.test.user] for setting [index.acc.test.pw]", iae.getMessage());
// user has no dependency
client()
.admin()
.indices()
.prepareUpdateSettings("test")
.setSettings(
Settings.builder()
.put("index.acc.test.user", "asdf"))
.execute()
.actionGet();
// now we are consistent
client()
.admin()
.indices()
.prepareUpdateSettings("test")
.setSettings(
Settings.builder()
.put("index.acc.test.pw", "test"))
.execute()
.actionGet();
// now try to remove it and make sure it fails
iae = expectThrows(IllegalArgumentException.class, () ->
client()
.admin()
.indices()
.prepareUpdateSettings("test")
.setSettings(
Settings.builder()
.putNull("index.acc.test.user"))
.execute()
.actionGet());
assertEquals("Missing required setting [index.acc.test.user] for setting [index.acc.test.pw]", iae.getMessage());
// now we are consistent
client()
.admin()
.indices()
.prepareUpdateSettings("test")
.setSettings(
Settings.builder()
.putNull("index.acc.test.pw")
.putNull("index.acc.test.user"))
.execute()
.actionGet();
}
}
public void testResetDefault() {
createIndex("test");

44
plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java
View File

@ -38,44 +38,47 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
public final class AzureStorageSettings {
// prefix for azure client settings
private static final String PREFIX = "azure.client.";
private static final String AZURE_CLIENT_PREFIX_KEY = "azure.client.";
/** Azure account name */
public static final AffixSetting<SecureString> ACCOUNT_SETTING =
Setting.affixKeySetting(PREFIX, "account", key -> SecureSetting.secureString(key, null));
Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "account", key -> SecureSetting.secureString(key, null));
/** Azure key */
public static final AffixSetting<SecureString> KEY_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "key",
key -> SecureSetting.secureString(key, null));
/** max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT). */
private static final Setting<Integer> MAX_RETRIES_SETTING =
Setting.affixKeySetting(PREFIX, "max_retries",
(key) -> Setting.intSetting(key, RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT, Setting.Property.NodeScope));
Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "max_retries",
(key) -> Setting.intSetting(key, RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT, Setting.Property.NodeScope),
ACCOUNT_SETTING, KEY_SETTING);
/**
* Azure endpoint suffix. Default to core.windows.net (CloudStorageAccount.DEFAULT_DNS).
*/
public static final Setting<String> ENDPOINT_SUFFIX_SETTING = Setting.affixKeySetting(PREFIX, "endpoint_suffix",
key -> Setting.simpleString(key, Property.NodeScope));
public static final Setting<String> ENDPOINT_SUFFIX_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "endpoint_suffix",
key -> Setting.simpleString(key, Property.NodeScope), ACCOUNT_SETTING, KEY_SETTING);
/** Azure key */
public static final AffixSetting<SecureString> KEY_SETTING = Setting.affixKeySetting(PREFIX, "key",
key -> SecureSetting.secureString(key, null));
public static final AffixSetting<TimeValue> TIMEOUT_SETTING = Setting.affixKeySetting(PREFIX, "timeout",
(key) -> Setting.timeSetting(key, TimeValue.timeValueMinutes(-1), Property.NodeScope));
public static final AffixSetting<TimeValue> TIMEOUT_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "timeout",
(key) -> Setting.timeSetting(key, TimeValue.timeValueMinutes(-1), Property.NodeScope), ACCOUNT_SETTING, KEY_SETTING);
/** The type of the proxy to connect to azure through. Can be direct (no proxy, default), http or socks */
public static final AffixSetting<Proxy.Type> PROXY_TYPE_SETTING = Setting.affixKeySetting(PREFIX, "proxy.type",
(key) -> new Setting<>(key, "direct", s -> Proxy.Type.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope));
public static final AffixSetting<Proxy.Type> PROXY_TYPE_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "proxy.type",
(key) -> new Setting<>(key, "direct", s -> Proxy.Type.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope)
, ACCOUNT_SETTING, KEY_SETTING);
/** The host name of a proxy to connect to azure through. */
public static final Setting<String> PROXY_HOST_SETTING = Setting.affixKeySetting(PREFIX, "proxy.host",
(key) -> Setting.simpleString(key, Property.NodeScope));
public static final AffixSetting<String> PROXY_HOST_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "proxy.host",
(key) -> Setting.simpleString(key, Property.NodeScope), KEY_SETTING, ACCOUNT_SETTING, PROXY_TYPE_SETTING);
/** The port of a proxy to connect to azure through. */
public static final Setting<Integer> PROXY_PORT_SETTING = Setting.affixKeySetting(PREFIX, "proxy.port",
(key) -> Setting.intSetting(key, 0, 0, 65535, Setting.Property.NodeScope));
public static final Setting<Integer> PROXY_PORT_SETTING = Setting.affixKeySetting(AZURE_CLIENT_PREFIX_KEY, "proxy.port",
(key) -> Setting.intSetting(key, 0, 0, 65535, Setting.Property.NodeScope), ACCOUNT_SETTING, KEY_SETTING, PROXY_TYPE_SETTING,
PROXY_HOST_SETTING);
private final String account;
private final String key;
@ -157,9 +160,8 @@ public final class AzureStorageSettings {
*/
public static Map<String, AzureStorageSettings> load(Settings settings) {
// Get the list of existing named configurations
Set<String> clientNames = settings.getGroups(PREFIX).keySet();
Map<String, AzureStorageSettings> storageSettings = new HashMap<>();
for (String clientName : clientNames) {
for (String clientName : ACCOUNT_SETTING.getNamespaces(settings)) {
storageSettings.put(clientName, getClientSettings(settings, clientName));
}

12
plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
View File

@ -40,6 +40,7 @@ import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.iterable.Iterables;
import org.elasticsearch.env.Environment;
import java.io.IOException;
@ -54,10 +55,8 @@ import java.util.Set;
interface GoogleCloudStorageService {
String SETTINGS_PREFIX = "gcs.client.";
/** A json credentials file loaded from secure settings. */
Setting.AffixSetting<InputStream> CREDENTIALS_FILE_SETTING = Setting.affixKeySetting(SETTINGS_PREFIX, "credentials_file",
Setting.AffixSetting<InputStream> CREDENTIALS_FILE_SETTING = Setting.affixKeySetting("gcs.client.", "credentials_file",
key -> SecureSetting.secureFile(key, null));
/**
@ -176,16 +175,15 @@ interface GoogleCloudStorageService {
/** Load all secure credentials from the settings. */
static Map<String, GoogleCredential> loadClientCredentials(Settings settings) {
Set<String> clientNames = settings.getGroups(SETTINGS_PREFIX).keySet();
Map<String, GoogleCredential> credentials = new HashMap<>();
for (String clientName : clientNames) {
Setting<InputStream> concreteSetting = CREDENTIALS_FILE_SETTING.getConcreteSettingForNamespace(clientName);
Iterable<Setting<InputStream>> iterable = CREDENTIALS_FILE_SETTING.getAllConcreteSettings(settings)::iterator;
for (Setting<InputStream> concreteSetting : iterable) {
try (InputStream credStream = concreteSetting.get(settings)) {
GoogleCredential credential = GoogleCredential.fromStream(credStream);
if (credential.createScopedRequired()) {
credential = credential.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL));
}
credentials.put(clientName, credential);
credentials.put(CREDENTIALS_FILE_SETTING.getNamespace(concreteSetting), credential);
} catch (IOException e) {
throw new UncheckedIOException(e);
}

2
test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
View File

@ -2568,7 +2568,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
Settings randomSettings = randomFrom(random(), globalSettings, transportSettings, profileSettings);
ClusterSettings clusterSettings = new ClusterSettings(randomSettings, ClusterSettings
.BUILT_IN_CLUSTER_SETTINGS);
clusterSettings.validate(randomSettings);
clusterSettings.validate(randomSettings, false);
TcpTransport.ProfileSettings settings = new TcpTransport.ProfileSettings(
Settings.builder().put(randomSettings).put("transport.profiles.some_profile.port", "9700-9800").build(), // port is required
"some_profile");