diff --git a/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java b/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java index 36d6b97a40e..21480fd1b1d 100644 --- a/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java +++ b/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java @@ -19,7 +19,6 @@ package org.apache.druid.data.input.impl; -import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; @@ -29,7 +28,6 @@ import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.InputSourceReader; import org.apache.druid.data.input.InputSplit; import org.apache.druid.data.input.SplitHintSpec; -import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.metadata.PasswordProvider; import javax.annotation.Nullable; @@ -48,25 +46,17 @@ public class HttpInputSource extends AbstractInputSource implements SplittableIn @Nullable private final PasswordProvider httpAuthenticationPasswordProvider; - private final HttpInputSourceConfig config; - @JsonCreator public HttpInputSource( @JsonProperty("uris") List uris, @JsonProperty("httpAuthenticationUsername") @Nullable String httpAuthenticationUsername, - @JsonProperty("httpAuthenticationPassword") @Nullable PasswordProvider httpAuthenticationPasswordProvider, - @JacksonInject HttpInputSourceConfig config + @JsonProperty("httpAuthenticationPassword") @Nullable PasswordProvider httpAuthenticationPasswordProvider ) { Preconditions.checkArgument(uris != null && !uris.isEmpty(), "Empty URIs"); - uris.forEach(uri -> Preconditions.checkArgument( - config.isURIAllowed(uri), - StringUtils.format("Access to [%s] DENIED!", uri) - )); this.uris = uris; this.httpAuthenticationUsername = httpAuthenticationUsername; this.httpAuthenticationPasswordProvider = httpAuthenticationPasswordProvider; - this.config = config; } @JsonProperty @@ -107,8 +97,7 @@ public class HttpInputSource extends AbstractInputSource implements SplittableIn return new HttpInputSource( Collections.singletonList(split.get()), httpAuthenticationUsername, - httpAuthenticationPasswordProvider, - config + httpAuthenticationPasswordProvider ); } diff --git a/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSourceConfig.java b/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSourceConfig.java deleted file mode 100644 index a84fb31e439..00000000000 --- a/core/src/main/java/org/apache/druid/data/input/impl/HttpInputSourceConfig.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.data.input.impl; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Preconditions; - -import java.net.URI; -import java.util.List; -import java.util.Objects; - -public class HttpInputSourceConfig -{ - @JsonProperty - private final List allowListDomains; - @JsonProperty - private final List denyListDomains; - - @JsonCreator - public HttpInputSourceConfig( - @JsonProperty("allowListDomains") List allowListDomains, - @JsonProperty("denyListDomains") List denyListDomains - ) - { - this.allowListDomains = allowListDomains; - this.denyListDomains = denyListDomains; - Preconditions.checkArgument( - this.denyListDomains == null || this.allowListDomains == null, - "Can only use one of allowList or blackList" - ); - } - - public List getAllowListDomains() - { - return allowListDomains; - } - - public List getDenyListDomains() - { - return denyListDomains; - } - - private static boolean matchesAny(List domains, URI uri) - { - for (String domain : domains) { - if (uri.getHost().endsWith(domain)) { - return true; - } - } - return false; - } - - public boolean isURIAllowed(URI uri) - { - if (allowListDomains != null) { - return matchesAny(allowListDomains, uri); - } - if (denyListDomains != null) { - return !matchesAny(denyListDomains, uri); - } - // No blacklist/allowList configured, all URLs are allowed - return true; - } - - @Override - public String toString() - { - return "HttpInputSourceConfig{" + - "allowListDomains=" + allowListDomains + - ", denyListDomains=" + denyListDomains + - '}'; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - HttpInputSourceConfig config = (HttpInputSourceConfig) o; - return Objects.equals(allowListDomains, config.allowListDomains) && - Objects.equals(denyListDomains, config.denyListDomains); - } - - @Override - public int hashCode() - { - return Objects.hash(allowListDomains, denyListDomains); - } -} - diff --git a/core/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java b/core/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java index 892fb38b1d8..61be2cc1b20 100644 --- a/core/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java +++ b/core/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java @@ -19,7 +19,6 @@ package org.apache.druid.data.input.impl; -import com.fasterxml.jackson.databind.InjectableValues; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import org.apache.druid.data.input.InputSource; @@ -29,91 +28,20 @@ import org.junit.Test; import java.io.IOException; import java.net.URI; -import java.util.Collections; public class HttpInputSourceTest { @Test public void testSerde() throws IOException { - HttpInputSourceConfig httpInputSourceConfig = new HttpInputSourceConfig( - null, - null - ); final ObjectMapper mapper = new ObjectMapper(); - mapper.setInjectableValues(new InjectableValues.Std().addValue( - HttpInputSourceConfig.class, - httpInputSourceConfig - )); final HttpInputSource source = new HttpInputSource( ImmutableList.of(URI.create("http://test.com/http-test")), "myName", - new DefaultPasswordProvider("myPassword"), - httpInputSourceConfig + new DefaultPasswordProvider("myPassword") ); final byte[] json = mapper.writeValueAsBytes(source); final HttpInputSource fromJson = (HttpInputSource) mapper.readValue(json, InputSource.class); Assert.assertEquals(source, fromJson); } - - @Test(expected = IllegalArgumentException.class) - public void testDenyListDomainThrowsException() - { - new HttpInputSource( - ImmutableList.of(URI.create("http://deny.com/http-test")), - "myName", - new DefaultPasswordProvider("myPassword"), - new HttpInputSourceConfig(null, Collections.singletonList("deny.com")) - ); - } - - @Test - public void testDenyListDomainNoMatch() - { - new HttpInputSource( - ImmutableList.of(URI.create("http://allow.com/http-test")), - "myName", - new DefaultPasswordProvider("myPassword"), - new HttpInputSourceConfig(null, Collections.singletonList("deny.com")) - ); - } - - @Test(expected = IllegalArgumentException.class) - public void testAllowListDomainThrowsException() - { - new HttpInputSource( - ImmutableList.of(URI.create("http://deny.com/http-test")), - "myName", - new DefaultPasswordProvider("myPassword"), - new HttpInputSourceConfig(Collections.singletonList("allow.com"), null) - ); - } - - @Test - public void testAllowListDomainMatch() - { - new HttpInputSource( - ImmutableList.of(URI.create("http://allow.com/http-test")), - "myName", - new DefaultPasswordProvider("myPassword"), - new HttpInputSourceConfig(Collections.singletonList("allow.com"), null) - ); - } - - @Test(expected = IllegalArgumentException.class) - public void testEmptyAllowListDomainMatch() - { - new HttpInputSource( - ImmutableList.of(URI.create("http://allow.com/http-test")), - "myName", - new DefaultPasswordProvider("myPassword"), - new HttpInputSourceConfig(Collections.emptyList(), null) - ); - } - - @Test(expected = IllegalArgumentException.class) - public void testCannotSetBothAllowAndDenyList() - { - new HttpInputSourceConfig(Collections.singletonList("allow.com"), Collections.singletonList("deny.com")); - } } diff --git a/docs/configuration/index.md b/docs/configuration/index.md index fbad3865bfb..8e924f4139b 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -1355,15 +1355,6 @@ The amount of direct memory needed by Druid is at least ensure at least this amount of direct memory is available by providing `-XX:MaxDirectMemorySize=` at the command line. -#### Indexer Security Configuration -You can optionally configure following additional configs to restrict druid ingestion - -|Property|Possible Values|Description|Default| -|--------|---------------|-----------|-------| -|`druid.ingestion.http.allowListDomains`|List of domains|Allowed domains from which ingestion will be allowed. Only one of allowList or denyList can be set.|empty list| -|`druid.ingestion.http.denyListDomains`|List of domains|Blacklisted domains from which ingestion will NOT be allowed. Only one of allowList or denyList can be set. |empty list| - - #### Query Configurations See [general query configuration](#general-query-configuration). diff --git a/server/src/main/java/org/apache/druid/metadata/input/InputSourceModule.java b/server/src/main/java/org/apache/druid/metadata/input/InputSourceModule.java index eb612f70958..0423af4f9a0 100644 --- a/server/src/main/java/org/apache/druid/metadata/input/InputSourceModule.java +++ b/server/src/main/java/org/apache/druid/metadata/input/InputSourceModule.java @@ -24,8 +24,6 @@ import com.fasterxml.jackson.databind.jsontype.NamedType; import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableList; import com.google.inject.Binder; -import org.apache.druid.data.input.impl.HttpInputSourceConfig; -import org.apache.druid.guice.JsonConfigProvider; import org.apache.druid.initialization.DruidModule; import java.util.List; @@ -49,6 +47,5 @@ public class InputSourceModule implements DruidModule @Override public void configure(Binder binder) { - JsonConfigProvider.bind(binder, "druid.ingestion.http", HttpInputSourceConfig.class); } } diff --git a/server/src/test/java/org/apache/druid/metadata/input/InputSourceModuleTest.java b/server/src/test/java/org/apache/druid/metadata/input/InputSourceModuleTest.java index 81fa5d3bb04..67126b0c7b2 100644 --- a/server/src/test/java/org/apache/druid/metadata/input/InputSourceModuleTest.java +++ b/server/src/test/java/org/apache/druid/metadata/input/InputSourceModuleTest.java @@ -25,27 +25,12 @@ import com.fasterxml.jackson.databind.cfg.MapperConfig; import com.fasterxml.jackson.databind.introspect.AnnotatedClass; import com.fasterxml.jackson.databind.introspect.AnnotatedClassResolver; import com.fasterxml.jackson.databind.jsontype.NamedType; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.ProvisionException; -import org.apache.druid.data.input.impl.HttpInputSourceConfig; -import org.apache.druid.guice.DruidGuiceExtensions; -import org.apache.druid.guice.JsonConfigurator; -import org.apache.druid.guice.LazySingleton; -import org.apache.druid.guice.LifecycleModule; -import org.apache.druid.guice.ServerModule; -import org.apache.druid.jackson.JacksonModule; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import javax.validation.Validation; -import javax.validation.Validator; -import java.util.Collections; import java.util.List; -import java.util.Properties; import java.util.stream.Collectors; public class InputSourceModuleTest @@ -56,8 +41,7 @@ public class InputSourceModuleTest @Before public void setUp() { - InputSourceModule inputSourceModule = new InputSourceModule(); - for (Module jacksonModule : inputSourceModule.getJacksonModules()) { + for (Module jacksonModule : new InputSourceModule().getJacksonModules()) { mapper.registerModule(jacksonModule); } } @@ -75,62 +59,4 @@ public class InputSourceModuleTest Assert.assertNotNull(subtypes); Assert.assertEquals(SQL_NAMED_TYPE, Iterables.getOnlyElement(subtypes)); } - - @Test - public void testHttpInputSourceAllowConfig() - { - Properties props = new Properties(); - props.put("druid.ingestion.http.allowListDomains", "[\"allow.com\"]"); - Injector injector = makeInjectorWithProperties(props); - HttpInputSourceConfig instance = injector.getInstance(HttpInputSourceConfig.class); - Assert.assertEquals(new HttpInputSourceConfig(Collections.singletonList("allow.com"), null), instance); - } - - @Test - public void testHttpInputSourceDenyConfig() - { - Properties props = new Properties(); - props.put("druid.ingestion.http.denyListDomains", "[\"deny.com\"]"); - Injector injector = makeInjectorWithProperties(props); - HttpInputSourceConfig instance = injector.getInstance(HttpInputSourceConfig.class); - Assert.assertEquals(new HttpInputSourceConfig(null, Collections.singletonList("deny.com")), instance); - } - - @Test(expected = ProvisionException.class) - public void testHttpInputSourceBothAllowDenyConfig() - { - Properties props = new Properties(); - props.put("druid.ingestion.http.allowListDomains", "[\"allow.com\"]"); - props.put("druid.ingestion.http.denyListDomains", "[\"deny.com\"]"); - Injector injector = makeInjectorWithProperties(props); - injector.getInstance(HttpInputSourceConfig.class); - } - - @Test - public void testHttpInputSourceDefaultConfig() - { - Properties props = new Properties(); - Injector injector = makeInjectorWithProperties(props); - HttpInputSourceConfig instance = injector.getInstance(HttpInputSourceConfig.class); - Assert.assertEquals(new HttpInputSourceConfig(null, null), instance); - Assert.assertNull(instance.getAllowListDomains()); - Assert.assertNull(instance.getDenyListDomains()); - } - - private Injector makeInjectorWithProperties(final Properties props) - { - return Guice.createInjector( - ImmutableList.of( - new DruidGuiceExtensions(), - new LifecycleModule(), - new ServerModule(), - binder -> { - binder.bind(Validator.class).toInstance(Validation.buildDefaultValidatorFactory().getValidator()); - binder.bind(JsonConfigurator.class).in(LazySingleton.class); - binder.bind(Properties.class).toInstance(props); - }, - new JacksonModule(), - new InputSourceModule() - )); - } } diff --git a/website/.spelling b/website/.spelling index 9c31e4b8c24..e4cdaac495c 100644 --- a/website/.spelling +++ b/website/.spelling @@ -1676,7 +1676,6 @@ _default_tier addr affinityConfig allowAll -allowList ANDed array_mod autoscale @@ -1690,7 +1689,6 @@ cpuacct dataSourceName datetime defaultHistory -denyList doubleMax doubleMin doubleSum