Add interpolation to JsonConfigurator (#13023)

* Add interpolation to JsonConfigurator

* Fix checkstyle

* Fix tests by removing common-text override

* Add back commons-text without version

* Remove unused hadoopDir configs

* Move some stuff to hopefully pass coverage
This commit is contained in:
Adam Peck 2022-09-07 01:18:01 -06:00 committed by GitHub
parent a3a377e570
commit ee22663dd3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 145 additions and 17 deletions

View File

@ -67,6 +67,10 @@
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId> <artifactId>commons-compress</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.skife.config</groupId> <groupId>org.skife.config</groupId>
<artifactId>config-magic</artifactId> <artifactId>config-magic</artifactId>
@ -368,6 +372,11 @@
<version>${postgresql.version}</version> <version>${postgresql.version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>com.github.stefanbirkner</groupId>
<artifactId>system-rules</artifactId>
<scope>test</scope>
</dependency>
</dependencies> </dependencies>
<build> <build>

View File

@ -27,10 +27,13 @@ import com.fasterxml.jackson.databind.introspect.BeanPropertyDefinition;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function; import com.google.common.base.Function;
import com.google.common.base.Strings; import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables; import com.google.common.collect.Iterables;
import com.google.inject.Inject; import com.google.inject.Inject;
import com.google.inject.ProvisionException; import com.google.inject.ProvisionException;
import com.google.inject.spi.Message; import com.google.inject.spi.Message;
import org.apache.commons.text.StringSubstitutor;
import org.apache.commons.text.lookup.StringLookupFactory;
import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.common.logger.Logger;
@ -58,6 +61,15 @@ public class JsonConfigurator
private final ObjectMapper jsonMapper; private final ObjectMapper jsonMapper;
private final Validator validator; private final Validator validator;
private final StringSubstitutor stringSubstitutor = new StringSubstitutor(StringLookupFactory.INSTANCE.interpolatorStringLookup(
ImmutableMap.of(
StringLookupFactory.KEY_SYS, StringLookupFactory.INSTANCE.systemPropertyStringLookup(),
StringLookupFactory.KEY_ENV, StringLookupFactory.INSTANCE.environmentVariableStringLookup(),
StringLookupFactory.KEY_FILE, StringLookupFactory.INSTANCE.fileStringLookup()
),
null,
false
)).setEnableSubstitutionInVariables(true).setEnableUndefinedVariableException(true);
@Inject @Inject
public JsonConfigurator( public JsonConfigurator(
@ -89,7 +101,7 @@ public class JsonConfigurator
Map<String, Object> jsonMap = new HashMap<>(); Map<String, Object> jsonMap = new HashMap<>();
for (String prop : props.stringPropertyNames()) { for (String prop : props.stringPropertyNames()) {
if (prop.startsWith(propertyBase)) { if (prop.startsWith(propertyBase)) {
final String propValue = props.getProperty(prop); final String propValue = stringSubstitutor.replace(props.getProperty(prop));
Object value; Object value;
try { try {
// If it's a String Jackson wants it to be quoted, so check if it's not an object or array and quote. // If it's a String Jackson wants it to be quoted, so check if it's not an object or array and quote.

View File

@ -27,7 +27,10 @@ import com.google.common.collect.ImmutableSet;
import org.apache.druid.TestObjectMapper; import org.apache.druid.TestObjectMapper;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.contrib.java.lang.system.EnvironmentVariables;
import org.junit.contrib.java.lang.system.RestoreSystemProperties;
import javax.validation.ConstraintViolation; import javax.validation.ConstraintViolation;
import javax.validation.Validator; import javax.validation.Validator;
@ -43,6 +46,12 @@ public class JsonConfiguratorTest
private final ObjectMapper mapper = new TestObjectMapper(); private final ObjectMapper mapper = new TestObjectMapper();
private final Properties properties = new Properties(); private final Properties properties = new Properties();
@Rule
public final RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties();
@Rule
public final EnvironmentVariables environmentVariables = new EnvironmentVariables();
@Before @Before
public void setUp() public void setUp()
{ {
@ -159,6 +168,71 @@ public class JsonConfiguratorTest
Assert.assertEquals("prop1", obj.prop1); Assert.assertEquals("prop1", obj.prop1);
} }
@Test
public void testPropertyInterpolation()
{
System.setProperty("my.property", "value1");
List<String> list = ImmutableList.of("list", "of", "strings");
environmentVariables.set("MY_VAR", "value2");
final JsonConfigurator configurator = new JsonConfigurator(mapper, validator);
properties.setProperty(PROP_PREFIX + "prop1", "${sys:my.property}");
properties.setProperty(PROP_PREFIX + "prop1List", "${file:UTF-8:src/test/resources/list.json}");
properties.setProperty(PROP_PREFIX + "prop2.prop.2", "${env:MY_VAR}");
final MappableObject obj = configurator.configurate(properties, PROP_PREFIX, MappableObject.class);
Assert.assertEquals(System.getProperty("my.property"), obj.prop1);
Assert.assertEquals(list, obj.prop1List);
Assert.assertEquals("value2", obj.prop2);
}
@Test
public void testPropertyInterpolationInName()
{
System.setProperty("my.property", "value1");
List<String> list = ImmutableList.of("list", "of", "strings");
environmentVariables.set("MY_VAR", "value2");
environmentVariables.set("SYS_PROP", "my.property");
System.setProperty("json.path", "src/test/resources/list.json");
environmentVariables.set("PROP2_NAME", "MY_VAR");
final JsonConfigurator configurator = new JsonConfigurator(mapper, validator);
properties.setProperty(PROP_PREFIX + "prop1", "${sys:${env:SYS_PROP}}");
properties.setProperty(PROP_PREFIX + "prop1List", "${file:UTF-8:${sys:json.path}}");
properties.setProperty(PROP_PREFIX + "prop2.prop.2", "${env:${env:PROP2_NAME}}");
final MappableObject obj = configurator.configurate(properties, PROP_PREFIX, MappableObject.class);
Assert.assertEquals(System.getProperty("my.property"), obj.prop1);
Assert.assertEquals(list, obj.prop1List);
Assert.assertEquals("value2", obj.prop2);
}
@Test
public void testPropertyInterpolationFallback()
{
List<String> list = ImmutableList.of("list", "of", "strings");
final JsonConfigurator configurator = new JsonConfigurator(mapper, validator);
properties.setProperty(PROP_PREFIX + "prop1", "${sys:my.property:-value1}");
properties.setProperty(PROP_PREFIX + "prop1List", "${unknown:-[\"list\", \"of\", \"strings\"]}");
properties.setProperty(PROP_PREFIX + "prop2.prop.2", "${MY_VAR:-value2}");
final MappableObject obj = configurator.configurate(properties, PROP_PREFIX, MappableObject.class);
Assert.assertEquals("value1", obj.prop1);
Assert.assertEquals(list, obj.prop1List);
Assert.assertEquals("value2", obj.prop2);
}
@Test
public void testPropertyInterpolationUndefinedException()
{
final JsonConfigurator configurator = new JsonConfigurator(mapper, validator);
properties.setProperty(PROP_PREFIX + "prop1", "${sys:my.property}");
Assert.assertThrows(
IllegalArgumentException.class,
() -> configurator.configurate(properties, PROP_PREFIX, MappableObject.class)
);
}
} }
class MappableObject class MappableObject

View File

@ -0,0 +1,5 @@
[
"list",
"of",
"strings"
]

View File

@ -61,6 +61,35 @@ The `jvm.config` files contain JVM flags such as heap sizing properties for each
Common properties shared by all services are placed in `_common/common.runtime.properties`. Common properties shared by all services are placed in `_common/common.runtime.properties`.
## Configuration Interpolation
Configuration values can be interpolated from System Properties, Environment Variables, or local files. Below is an example of how this can be used:
```
druid.metadata.storage.type=${env:METADATA_STORAGE_TYPE}
druid.processing.tmpDir=${sys:java.io.tmpdir}
druid.segmentCache.locations=${file:UTF-8:/config/segment-cache-def.json}
```
Interpolation is also recursive so you can do:
```
druid.segmentCache.locations=${file:UTF-8:${env:SEGMENT_DEF_LOCATION}}
```
If the property is not set an exception will be thrown on startup, but a default can be provided if desired. Setting a default value will not work with file interpolation as an exception will be thrown if the file does not exist.
```
druid.metadata.storage.type=${env:METADATA_STORAGE_TYPE:-mysql}
druid.processing.tmpDir=${sys:java.io.tmpdir:-/tmp}
```
If you need to set a variable that is wrapped by `${...}` but do not want it to be interpolated you can escape it by adding another `$`. For example:
```
config.name=$${value}
```
## Common Configurations ## Common Configurations
The properties under this section are common configurations that should be shared across all Druid services in a cluster. The properties under this section are common configurations that should be shared across all Druid services in a cluster.

View File

@ -260,7 +260,6 @@ public class Initializer
// previously set in Maven. // previously set in Maven.
propertyEnvVarBinding("druid.test.config.dockerIp", "DOCKER_IP"); propertyEnvVarBinding("druid.test.config.dockerIp", "DOCKER_IP");
propertyEnvVarBinding("druid.zk.service.host", "DOCKER_IP"); propertyEnvVarBinding("druid.zk.service.host", "DOCKER_IP");
propertyEnvVarBinding("druid.test.config.hadoopDir", "HADOOP_DIR");
property("druid.client.https.trustStorePath", "client_tls/truststore.jks"); property("druid.client.https.trustStorePath", "client_tls/truststore.jks");
property("druid.client.https.trustStorePassword", "druid123"); property("druid.client.https.trustStorePassword", "druid123");
property("druid.client.https.keyStorePath", "client_tls/client.jks"); property("druid.client.https.keyStorePath", "client_tls/client.jks");

View File

@ -651,7 +651,6 @@
-Duser.timezone=UTC -Duser.timezone=UTC
-Dfile.encoding=UTF-8 -Dfile.encoding=UTF-8
-Ddruid.test.config.dockerIp=${env.DOCKER_IP} -Ddruid.test.config.dockerIp=${env.DOCKER_IP}
-Ddruid.test.config.hadoopDir=${env.HADOOP_DIR}
-Ddruid.test.config.extraDatasourceNameSuffix=${extra.datasource.name.suffix} -Ddruid.test.config.extraDatasourceNameSuffix=${extra.datasource.name.suffix}
-Ddruid.zk.service.host=${env.DOCKER_IP} -Ddruid.zk.service.host=${env.DOCKER_IP}
-Ddruid.client.https.trustStorePath=client_tls/truststore.jks -Ddruid.client.https.trustStorePath=client_tls/truststore.jks

View File

@ -668,13 +668,16 @@ name: Apache Commons Lang
license_category: binary license_category: binary
module: java-core module: java-core
license_name: Apache License version 2.0 license_name: Apache License version 2.0
version: 3.8.1 version: 3.12.0
libraries: libraries:
- org.apache.commons: commons-lang3 - org.apache.commons: commons-lang3
notices: notices:
- commons-lang3: | - commons-lang3: |
Apache Commons Lang Apache Commons Lang
Copyright 2001-2018 The Apache Software Foundation Copyright 2001-2021 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (https://www.apache.org/).
--- ---
@ -719,23 +722,17 @@ name: Apache Commons Text
license_category: binary license_category: binary
module: java-core module: java-core
license_name: Apache License version 2.0 license_name: Apache License version 2.0
version: 1.3 version: 1.9
libraries: libraries:
- org.apache.commons: commons-text - org.apache.commons: commons-text
notices: notices:
- commons-text: | - commons-text: |
Apache Commons Text Apache Commons Text
Copyright 2001-2018 The Apache Software Foundation Copyright 2014-2020 The Apache Software Foundation
--- This product includes software developed at
The Apache Software Foundation (https://www.apache.org/).
name: Apache Commons Text
license_category: binary
module: java-core
license_name: Apache License version 2.0
version: 1.4
libraries:
- org.apache.commons: commons-text
--- ---
name: Airline name: Airline

View File

@ -270,7 +270,12 @@
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId> <artifactId>commons-lang3</artifactId>
<version>3.8.1</version> <version>3.12.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
<version>1.9</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.amazonaws</groupId> <groupId>com.amazonaws</groupId>

View File

@ -448,7 +448,6 @@
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId> <artifactId>commons-text</artifactId>
<version>1.3</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
</dependencies> </dependencies>