Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert "Add support for Blacklisting some domains for HTTPInputSource (#10535)" (#10871) 

This reverts commit 6b14bdb3a5.
This commit is contained in:
Jihoon Son 2021-02-09 17:51:26 -08:00 committed by GitHub
parent 41a0c1d4b5
commit 1ec3f0bd73
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 4 additions and 287 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
core/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java
View File

@ -19,7 +19,6 @@
package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
@ -29,7 +28,6 @@ import org.apache.druid.data.input.InputRowSchema;
import org.apache.druid.data.input.InputSourceReader;
import org.apache.druid.data.input.InputSplit;
import org.apache.druid.data.input.SplitHintSpec;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.metadata.PasswordProvider;
import javax.annotation.Nullable;
@ -48,25 +46,17 @@ public class HttpInputSource extends AbstractInputSource implements SplittableIn
@Nullable
private final PasswordProvider httpAuthenticationPasswordProvider;
private final HttpInputSourceConfig config;
@JsonCreator
public HttpInputSource(
@JsonProperty("uris") List<URI> uris,
@JsonProperty("httpAuthenticationUsername") @Nullable String httpAuthenticationUsername,
@JsonProperty("httpAuthenticationPassword") @Nullable PasswordProvider httpAuthenticationPasswordProvider,
@JacksonInject HttpInputSourceConfig config
@JsonProperty("httpAuthenticationPassword") @Nullable PasswordProvider httpAuthenticationPasswordProvider
)
{
Preconditions.checkArgument(uris != null && !uris.isEmpty(), "Empty URIs");
uris.forEach(uri -> Preconditions.checkArgument(
config.isURIAllowed(uri),
StringUtils.format("Access to [%s] DENIED!", uri)
));
this.uris = uris;
this.httpAuthenticationUsername = httpAuthenticationUsername;
this.httpAuthenticationPasswordProvider = httpAuthenticationPasswordProvider;
this.config = config;
}
@JsonProperty
@ -107,8 +97,7 @@ public class HttpInputSource extends AbstractInputSource implements SplittableIn
return new HttpInputSource(
Collections.singletonList(split.get()),
httpAuthenticationUsername,
httpAuthenticationPasswordProvider,
config
httpAuthenticationPasswordProvider
);
}

112
core/src/main/java/org/apache/druid/data/input/impl/HttpInputSourceConfig.java
View File

@ -1,112 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import java.net.URI;
import java.util.List;
import java.util.Objects;
public class HttpInputSourceConfig
{
@JsonProperty
private final List<String> allowListDomains;
@JsonProperty
private final List<String> denyListDomains;
@JsonCreator
public HttpInputSourceConfig(
@JsonProperty("allowListDomains") List<String> allowListDomains,
@JsonProperty("denyListDomains") List<String> denyListDomains
)
{
this.allowListDomains = allowListDomains;
this.denyListDomains = denyListDomains;
Preconditions.checkArgument(
this.denyListDomains == null || this.allowListDomains == null,
"Can only use one of allowList or blackList"
);
}
public List<String> getAllowListDomains()
{
return allowListDomains;
}
public List<String> getDenyListDomains()
{
return denyListDomains;
}
private static boolean matchesAny(List<String> domains, URI uri)
{
for (String domain : domains) {
if (uri.getHost().endsWith(domain)) {
return true;
}
}
return false;
}
public boolean isURIAllowed(URI uri)
{
if (allowListDomains != null) {
return matchesAny(allowListDomains, uri);
}
if (denyListDomains != null) {
return !matchesAny(denyListDomains, uri);
}
// No blacklist/allowList configured, all URLs are allowed
return true;
}
@Override
public String toString()
{
return "HttpInputSourceConfig{" +
"allowListDomains=" + allowListDomains +
", denyListDomains=" + denyListDomains +
'}';
}
@Override
public boolean equals(Object o)
{
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
HttpInputSourceConfig config = (HttpInputSourceConfig) o;
return Objects.equals(allowListDomains, config.allowListDomains) &&
Objects.equals(denyListDomains, config.denyListDomains);
}
@Override
public int hashCode()
{
return Objects.hash(allowListDomains, denyListDomains);
}
}

74
core/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java
View File

@ -19,7 +19,6 @@
package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.databind.InjectableValues;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import org.apache.druid.data.input.InputSource;
@ -29,91 +28,20 @@ import org.junit.Test;
import java.io.IOException;
import java.net.URI;
import java.util.Collections;
public class HttpInputSourceTest
{
@Test
public void testSerde() throws IOException
{
HttpInputSourceConfig httpInputSourceConfig = new HttpInputSourceConfig(
null,
null
);
final ObjectMapper mapper = new ObjectMapper();
mapper.setInjectableValues(new InjectableValues.Std().addValue(
HttpInputSourceConfig.class,
httpInputSourceConfig
));
final HttpInputSource source = new HttpInputSource(
ImmutableList.of(URI.create("http://test.com/http-test")),
"myName",
new DefaultPasswordProvider("myPassword"),
httpInputSourceConfig
new DefaultPasswordProvider("myPassword")
);
final byte[] json = mapper.writeValueAsBytes(source);
final HttpInputSource fromJson = (HttpInputSource) mapper.readValue(json, InputSource.class);
Assert.assertEquals(source, fromJson);
}
@Test(expected = IllegalArgumentException.class)
public void testDenyListDomainThrowsException()
{
new HttpInputSource(
ImmutableList.of(URI.create("http://deny.com/http-test")),
"myName",
new DefaultPasswordProvider("myPassword"),
new HttpInputSourceConfig(null, Collections.singletonList("deny.com"))
);
}
@Test
public void testDenyListDomainNoMatch()
{
new HttpInputSource(
ImmutableList.of(URI.create("http://allow.com/http-test")),
"myName",
new DefaultPasswordProvider("myPassword"),
new HttpInputSourceConfig(null, Collections.singletonList("deny.com"))
);
}
@Test(expected = IllegalArgumentException.class)
public void testAllowListDomainThrowsException()
{
new HttpInputSource(
ImmutableList.of(URI.create("http://deny.com/http-test")),
"myName",
new DefaultPasswordProvider("myPassword"),
new HttpInputSourceConfig(Collections.singletonList("allow.com"), null)
);
}
@Test
public void testAllowListDomainMatch()
{
new HttpInputSource(
ImmutableList.of(URI.create("http://allow.com/http-test")),
"myName",
new DefaultPasswordProvider("myPassword"),
new HttpInputSourceConfig(Collections.singletonList("allow.com"), null)
);
}
@Test(expected = IllegalArgumentException.class)
public void testEmptyAllowListDomainMatch()
{
new HttpInputSource(
ImmutableList.of(URI.create("http://allow.com/http-test")),
"myName",
new DefaultPasswordProvider("myPassword"),
new HttpInputSourceConfig(Collections.emptyList(), null)
);
}
@Test(expected = IllegalArgumentException.class)
public void testCannotSetBothAllowAndDenyList()
{
new HttpInputSourceConfig(Collections.singletonList("allow.com"), Collections.singletonList("deny.com"));
}
}

9
docs/configuration/index.md
View File

@ -1355,15 +1355,6 @@ The amount of direct memory needed by Druid is at least
ensure at least this amount of direct memory is available by providing `-XX:MaxDirectMemorySize=<VALUE>` at the command
line.
#### Indexer Security Configuration
You can optionally configure following additional configs to restrict druid ingestion
|Property|Possible Values|Description|Default|
|--------|---------------|-----------|-------|
|`druid.ingestion.http.allowListDomains`|List of domains|Allowed domains from which ingestion will be allowed. Only one of allowList or denyList can be set.|empty list|
|`druid.ingestion.http.denyListDomains`|List of domains|Blacklisted domains from which ingestion will NOT be allowed. Only one of allowList or denyList can be set. |empty list|
#### Query Configurations
See [general query configuration](#general-query-configuration).

3
server/src/main/java/org/apache/druid/metadata/input/InputSourceModule.java
View File

@ -24,8 +24,6 @@ import com.fasterxml.jackson.databind.jsontype.NamedType;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.google.common.collect.ImmutableList;
import com.google.inject.Binder;
import org.apache.druid.data.input.impl.HttpInputSourceConfig;
import org.apache.druid.guice.JsonConfigProvider;
import org.apache.druid.initialization.DruidModule;
import java.util.List;
@ -49,6 +47,5 @@ public class InputSourceModule implements DruidModule
@Override
public void configure(Binder binder)
{
JsonConfigProvider.bind(binder, "druid.ingestion.http", HttpInputSourceConfig.class);
}
}

76
server/src/test/java/org/apache/druid/metadata/input/InputSourceModuleTest.java
View File

@ -25,27 +25,12 @@ import com.fasterxml.jackson.databind.cfg.MapperConfig;
import com.fasterxml.jackson.databind.introspect.AnnotatedClass;
import com.fasterxml.jackson.databind.introspect.AnnotatedClassResolver;
import com.fasterxml.jackson.databind.jsontype.NamedType;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.ProvisionException;
import org.apache.druid.data.input.impl.HttpInputSourceConfig;
import org.apache.druid.guice.DruidGuiceExtensions;
import org.apache.druid.guice.JsonConfigurator;
import org.apache.druid.guice.LazySingleton;
import org.apache.druid.guice.LifecycleModule;
import org.apache.druid.guice.ServerModule;
import org.apache.druid.jackson.JacksonModule;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import javax.validation.Validation;
import javax.validation.Validator;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.stream.Collectors;
public class InputSourceModuleTest
@ -56,8 +41,7 @@ public class InputSourceModuleTest
@Before
public void setUp()
{
InputSourceModule inputSourceModule = new InputSourceModule();
for (Module jacksonModule : inputSourceModule.getJacksonModules()) {
for (Module jacksonModule : new InputSourceModule().getJacksonModules()) {
mapper.registerModule(jacksonModule);
}
}
@ -75,62 +59,4 @@ public class InputSourceModuleTest
Assert.assertNotNull(subtypes);
Assert.assertEquals(SQL_NAMED_TYPE, Iterables.getOnlyElement(subtypes));
}
@Test
public void testHttpInputSourceAllowConfig()
{
Properties props = new Properties();
props.put("druid.ingestion.http.allowListDomains", "[\"allow.com\"]");
Injector injector = makeInjectorWithProperties(props);
HttpInputSourceConfig instance = injector.getInstance(HttpInputSourceConfig.class);
Assert.assertEquals(new HttpInputSourceConfig(Collections.singletonList("allow.com"), null), instance);
}
@Test
public void testHttpInputSourceDenyConfig()
{
Properties props = new Properties();
props.put("druid.ingestion.http.denyListDomains", "[\"deny.com\"]");
Injector injector = makeInjectorWithProperties(props);
HttpInputSourceConfig instance = injector.getInstance(HttpInputSourceConfig.class);
Assert.assertEquals(new HttpInputSourceConfig(null, Collections.singletonList("deny.com")), instance);
}
@Test(expected = ProvisionException.class)
public void testHttpInputSourceBothAllowDenyConfig()
{
Properties props = new Properties();
props.put("druid.ingestion.http.allowListDomains", "[\"allow.com\"]");
props.put("druid.ingestion.http.denyListDomains", "[\"deny.com\"]");
Injector injector = makeInjectorWithProperties(props);
injector.getInstance(HttpInputSourceConfig.class);
}
@Test
public void testHttpInputSourceDefaultConfig()
{
Properties props = new Properties();
Injector injector = makeInjectorWithProperties(props);
HttpInputSourceConfig instance = injector.getInstance(HttpInputSourceConfig.class);
Assert.assertEquals(new HttpInputSourceConfig(null, null), instance);
Assert.assertNull(instance.getAllowListDomains());
Assert.assertNull(instance.getDenyListDomains());
}
private Injector makeInjectorWithProperties(final Properties props)
{
return Guice.createInjector(
ImmutableList.of(
new DruidGuiceExtensions(),
new LifecycleModule(),
new ServerModule(),
binder -> {
binder.bind(Validator.class).toInstance(Validation.buildDefaultValidatorFactory().getValidator());
binder.bind(JsonConfigurator.class).in(LazySingleton.class);
binder.bind(Properties.class).toInstance(props);
},
new JacksonModule(),
new InputSourceModule()
));
}
}

2
website/.spelling
View File

@ -1676,7 +1676,6 @@ _default_tier
addr
affinityConfig
allowAll
allowList
ANDed
array_mod
autoscale
@ -1690,7 +1689,6 @@ cpuacct
dataSourceName
datetime
defaultHistory
denyList
doubleMax
doubleMin
doubleSum