Add Environment Variable DynamicConfigProvider (#11377)

* add_environment_variable_DynamicConfigProvider

* fix code

* code fixed

* code fixed

* add document

* fix doc

* fix doc

* add more unit test

* fix style

* fix document

* bug fixed

* fix unit test

* fix comment

* fix test

Co-authored-by: yuanyi <yuanyi@freewheel.tv>

core/src/main/java/org/apache/druid/metadata/DynamicConfigProvider.java

@@ -32,6 +32,7 @@ import java.util.Map;
 @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = MapStringDynamicConfigProvider.class)
 @JsonSubTypes(value = {
     @JsonSubTypes.Type(name = "mapString", value = MapStringDynamicConfigProvider.class),
+    @JsonSubTypes.Type(name = "environment", value = EnvironmentVariableDynamicConfigProvider.class)
 })
 public interface DynamicConfigProvider<T>
 {

core/src/main/java/org/apache/druid/metadata/EnvironmentVariableDynamicConfigProvider.java

@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.metadata;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+public class EnvironmentVariableDynamicConfigProvider implements DynamicConfigProvider<String>
+{
+  private final ImmutableMap<String, String> variables;
+
+  @JsonCreator
+  public EnvironmentVariableDynamicConfigProvider(
+      @JsonProperty("variables") Map<String, String> config
+  )
+  {
+    this.variables = ImmutableMap.copyOf(Preconditions.checkNotNull(config, "config"));
+  }
+
+  @JsonProperty("variables")
+  public Map<String, String> getVariables()
+  {
+    return variables;
+  }
+
+  @Override
+  public Map<String, String> getConfig()
+  {
+    HashMap<String, String> map = new HashMap<>();
+    for (Map.Entry<String, String> entry : variables.entrySet()) {
+      map.put(entry.getKey(), System.getenv(entry.getValue()));
+    }
+    return map;
+  }
+
+  @Override
+  public String toString()
+  {
+    return "EnvironmentVariablePasswordProvider{" +
+        "variable='" + variables + '\'' +
+        '}';
+  }
+
+  @Override
+  public boolean equals(Object o)
+  {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+
+    EnvironmentVariableDynamicConfigProvider that = (EnvironmentVariableDynamicConfigProvider) o;
+
+    return Objects.equals(variables, that.variables);
+
+  }
+
+  @Override
+  public int hashCode()
+  {
+    return variables.hashCode();
+  }
+
+}

core/src/test/java/org/apache/druid/metadata/EnvironmentVariableDynamicConfigProviderTest.java

@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.metadata;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableMap;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class EnvironmentVariableDynamicConfigProviderTest
+{
+  private static final ObjectMapper JSON_MAPPER = new ObjectMapper();
+  private static final Map<String, String> CHANGED_ENV_MAP = new HashMap<>();
+
+  @BeforeClass
+  public static void setupTest() throws Exception
+  {
+    Map<String, String> oldEnvMap = getENVMap();
+    Map<String, String> addEnvMap = ImmutableMap.of("DRUID_USER", "druid", "DRUID_PASSWORD", "123");
+    for (Map.Entry<String, String> entry : addEnvMap.entrySet()) {
+      CHANGED_ENV_MAP.put(entry.getKey(), oldEnvMap.get(entry.getKey()));
+      oldEnvMap.put(entry.getKey(), entry.getValue());
+    }
+  }
+
+  @AfterClass
+  public static void tearDownTest() throws Exception
+  {
+    Map<String, String> oldEnvMap = getENVMap();
+    for (Map.Entry<String, String> entry : CHANGED_ENV_MAP.entrySet()) {
+      if (entry.getValue() == null) {
+        oldEnvMap.remove(entry.getKey());
+      } else {
+        oldEnvMap.put(entry.getKey(), entry.getValue());
+      }
+    }
+  }
+
+  @Test
+  public void testSerde() throws IOException
+  {
+    String providerString = "{\"type\": \"environment\", \"variables\" : {\"testKey\":\"testValue\"}}";
+    DynamicConfigProvider provider = JSON_MAPPER.readValue(providerString, DynamicConfigProvider.class);
+    Assert.assertTrue(provider instanceof EnvironmentVariableDynamicConfigProvider);
+    Assert.assertEquals("testValue", ((EnvironmentVariableDynamicConfigProvider) provider).getVariables().get("testKey"));
+    DynamicConfigProvider serde = JSON_MAPPER.readValue(JSON_MAPPER.writeValueAsString(provider), DynamicConfigProvider.class);
+    Assert.assertEquals(provider, serde);
+  }
+
+  @Test
+  public void testGetConfig() throws Exception
+  {
+    String providerString = "{\"type\": \"environment\", \"variables\" : {\"user\":\"DRUID_USER\",\"password\":\"DRUID_PASSWORD\"}}";
+    DynamicConfigProvider provider = JSON_MAPPER.readValue(providerString, DynamicConfigProvider.class);
+    Assert.assertTrue(provider instanceof EnvironmentVariableDynamicConfigProvider);
+    Assert.assertEquals("druid", ((EnvironmentVariableDynamicConfigProvider) provider).getConfig().get("user"));
+    Assert.assertEquals("123", ((EnvironmentVariableDynamicConfigProvider) provider).getConfig().get("password"));
+  }
+
+  /**
+   * This method use reflection to get system environment variables map in runtime JVM
+   * which can be changed.
+   *
+   * @return system environment variables map.
+   */
+  private static Map<String, String> getENVMap() throws Exception
+  {
+    Map<String, String> envMap = null;
+    Class[] classes = Collections.class.getDeclaredClasses();
+    Map<String, String> systemEnv = System.getenv();
+    for (Class cl : classes) {
+      if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) {
+        Field field = cl.getDeclaredField("m");
+        field.setAccessible(true);
+        Object object = field.get(systemEnv);
+        envMap = (Map<String, String>) object;
+      }
+    }
+    if (envMap == null) {
+      throw new RuntimeException("Failed to get environment map.");
+    }
+    return envMap;
+  }
+}

docs/operations/dynamic-config-provider.md

@@ -31,3 +31,16 @@ Users can create custom extension of the `DynamicConfigProvider` interface that
 
 For more information, see [Adding a new DynamicConfigProvider implementation](../development/modules.md#adding-a-new-dynamicconfigprovider-implementation).
 
+## Environment variable dynamic config provider
+
+`EnvironmentVariableDynamicConfigProvider` can be used to avoid exposing credentials or other secret information in the configuration files using environment variables. An example to use this `configProvider` is:
+```json
+druid.some.config.dynamicConfigProvider={"type": "environment","variables":{"secret1": "SECRET1_VAR","secret2": "SECRET2_VAR"}}
+```
+The values are described below.
+
+|Field|Type|Description|Required|
+|-----|----|-----------|--------|
+|`type`|String|dynamic config provider type|Yes: `environment`|
+|`variables`|Map|environment variables to get information from|Yes|
+