mirror of
https://github.com/apache/druid.git
synced 2025-02-17 23:46:30 +00:00
Update doc for allowedHeaders (#17045)
Update doc for allowedHeaders and make allowedHeaders more restrictive
This commit is contained in:
Pranav
GitHub
parent
2d2882cdfe
commit
d1bd6a8156
@ -616,9 +616,10 @@ the [HDFS input source](../ingestion/input-sources.md#hdfs-input-source).
|
||||
You can set the following property to specify permissible protocols for
|
||||
the [HTTP input source](../ingestion/input-sources.md#http-input-source).
|
||||
|
||||
|Property|Possible values|Description|Default|
|
||||
|--------|---------------|-----------|-------|
|
||||
|`druid.ingestion.http.allowedProtocols`|List of protocols|Allowed protocols for the HTTP input source.|`["http", "https"]`|
|
||||
|Property| Possible values | Description |Default|
|
||||
|--------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|-------|
|
||||
|`druid.ingestion.http.allowedProtocols`| List of protocols | Allowed protocols for the HTTP input source. |`["http", "https"]`|
|
||||
|`druid.ingestion.http.allowedHeaders`| A list of permitted request headers for the HTTP input source. By default, the list is empty, which means no headers are allowed in the ingestion specification. |`[]`|
|
||||
|
||||
### External data access security configuration
|
||||
|
||||
|
||||
@ -100,13 +100,11 @@ public class HttpInputSource
|
||||
|
||||
public static void throwIfForbiddenHeaders(HttpInputSourceConfig config, Map<String, String> requestHeaders)
|
||||
{
|
||||
if (config.getAllowedHeaders().size() > 0) {
|
||||
for (Map.Entry<String, String> entry : requestHeaders.entrySet()) {
|
||||
if (!config.getAllowedHeaders().contains(StringUtils.toLowerCase(entry.getKey()))) {
|
||||
throw InvalidInput.exception("Got forbidden header %s, allowed headers are only %s ",
|
||||
entry.getKey(), config.getAllowedHeaders()
|
||||
);
|
||||
}
|
||||
for (Map.Entry<String, String> entry : requestHeaders.entrySet()) {
|
||||
if (!config.getAllowedHeaders().contains(StringUtils.toLowerCase(entry.getKey()))) {
|
||||
throw InvalidInput.exception("Got forbidden header [%s], allowed headers are only [%s]. You can control the allowed headers by updating druid.ingestion.http.allowedHeaders",
|
||||
entry.getKey(), config.getAllowedHeaders()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -30,7 +30,6 @@ import org.apache.druid.data.input.InputSource;
|
||||
import org.apache.druid.data.input.impl.systemfield.SystemField;
|
||||
import org.apache.druid.data.input.impl.systemfield.SystemFields;
|
||||
import org.apache.druid.error.DruidException;
|
||||
import org.apache.druid.java.util.common.StringUtils;
|
||||
import org.apache.druid.metadata.DefaultPasswordProvider;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Rule;
|
||||
@ -41,8 +40,7 @@ import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.util.Collections;
|
||||
import java.util.EnumSet;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.HashSet;
|
||||
|
||||
public class HttpInputSourceTest
|
||||
{
|
||||
@ -152,12 +150,17 @@ public class HttpInputSourceTest
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAllowedHeaders()
|
||||
public void testEmptyAllowedHeaders()
|
||||
{
|
||||
HttpInputSourceConfig httpInputSourceConfig = new HttpInputSourceConfig(
|
||||
null,
|
||||
Sets.newHashSet("R-cookie", "Content-type")
|
||||
new HashSet<>()
|
||||
);
|
||||
expectedException.expect(DruidException.class);
|
||||
expectedException.expectMessage(
|
||||
"Got forbidden header [r-Cookie], allowed headers are only [[]]. "
|
||||
+ "You can control the allowed headers by updating druid.ingestion.http.allowedHeaders");
|
||||
|
||||
final HttpInputSource inputSource = new HttpInputSource(
|
||||
ImmutableList.of(URI.create("http://test.com/http-test")),
|
||||
"myName",
|
||||
@ -166,12 +169,6 @@ public class HttpInputSourceTest
|
||||
ImmutableMap.of("r-Cookie", "test", "Content-Type", "application/json"),
|
||||
httpInputSourceConfig
|
||||
);
|
||||
Set<String> expectedSet = inputSource.getRequestHeaders()
|
||||
.keySet()
|
||||
.stream()
|
||||
.map(StringUtils::toLowerCase)
|
||||
.collect(Collectors.toSet());
|
||||
Assert.assertEquals(expectedSet, httpInputSourceConfig.getAllowedHeaders());
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -183,7 +180,7 @@ public class HttpInputSourceTest
|
||||
);
|
||||
expectedException.expect(DruidException.class);
|
||||
expectedException.expectMessage(
|
||||
"Got forbidden header G-Cookie, allowed headers are only [r-cookie, content-type]");
|
||||
"Got forbidden header [G-Cookie], allowed headers are only [[r-cookie, content-type]]");
|
||||
new HttpInputSource(
|
||||
ImmutableList.of(URI.create("http://test.com/http-test")),
|
||||
"myName",
|
||||
|
||||
@ -20,9 +20,14 @@
|
||||
package org.apache.druid.sql.calcite;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.Module;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.module.SimpleModule;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Sets;
|
||||
import com.google.inject.Binder;
|
||||
import org.apache.calcite.avatica.SqlType;
|
||||
import org.apache.druid.catalog.model.Columns;
|
||||
import org.apache.druid.data.input.impl.CsvInputFormat;
|
||||
@ -31,20 +36,28 @@ import org.apache.druid.data.input.impl.HttpInputSourceConfig;
|
||||
import org.apache.druid.data.input.impl.JsonInputFormat;
|
||||
import org.apache.druid.data.input.impl.LocalInputSource;
|
||||
import org.apache.druid.data.input.impl.systemfield.SystemFields;
|
||||
import org.apache.druid.guice.DruidInjectorBuilder;
|
||||
import org.apache.druid.initialization.DruidModule;
|
||||
import org.apache.druid.java.util.common.ISE;
|
||||
import org.apache.druid.java.util.common.StringUtils;
|
||||
import org.apache.druid.java.util.common.UOE;
|
||||
import org.apache.druid.metadata.DefaultPasswordProvider;
|
||||
import org.apache.druid.metadata.input.InputSourceModule;
|
||||
import org.apache.druid.segment.column.ColumnType;
|
||||
import org.apache.druid.segment.column.RowSignature;
|
||||
import org.apache.druid.server.security.Access;
|
||||
import org.apache.druid.server.security.AuthConfig;
|
||||
import org.apache.druid.server.security.ForbiddenException;
|
||||
import org.apache.druid.sql.calcite.external.ExternalDataSource;
|
||||
import org.apache.druid.sql.calcite.external.ExternalOperatorConversion;
|
||||
import org.apache.druid.sql.calcite.external.Externals;
|
||||
import org.apache.druid.sql.calcite.external.HttpOperatorConversion;
|
||||
import org.apache.druid.sql.calcite.external.InlineOperatorConversion;
|
||||
import org.apache.druid.sql.calcite.external.LocalOperatorConversion;
|
||||
import org.apache.druid.sql.calcite.filtration.Filtration;
|
||||
import org.apache.druid.sql.calcite.planner.Calcites;
|
||||
import org.apache.druid.sql.calcite.util.CalciteTests;
|
||||
import org.apache.druid.sql.guice.SqlBindings;
|
||||
import org.apache.druid.sql.http.SqlParameter;
|
||||
import org.hamcrest.CoreMatchers;
|
||||
import org.junit.internal.matchers.ThrowableMessageMatcher;
|
||||
@ -53,8 +66,10 @@ import org.junit.jupiter.api.Test;
|
||||
import java.io.File;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import static org.hamcrest.MatcherAssert.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
@ -69,6 +84,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
* query ensure that the resulting MSQ task is identical regardless of the path
|
||||
* taken.
|
||||
*/
|
||||
@SqlTestFrameworkConfig.ComponentSupplier(IngestTableFunctionTest.ExportComponentSupplier.class)
|
||||
public class IngestTableFunctionTest extends CalciteIngestionDmlTest
|
||||
{
|
||||
protected static URI toURI(String uri)
|
||||
@ -97,6 +113,20 @@ public class IngestTableFunctionTest extends CalciteIngestionDmlTest
|
||||
.add("z", ColumnType.LONG)
|
||||
.build()
|
||||
);
|
||||
protected final ExternalDataSource localDataSource = new ExternalDataSource(
|
||||
new LocalInputSource(
|
||||
null,
|
||||
null,
|
||||
Arrays.asList(new File("/tmp/foo.csv"), new File("/tmp/bar.csv")),
|
||||
SystemFields.none()
|
||||
),
|
||||
new CsvInputFormat(ImmutableList.of("x", "y", "z"), null, false, false, 0),
|
||||
RowSignature.builder()
|
||||
.add("x", ColumnType.STRING)
|
||||
.add("y", ColumnType.STRING)
|
||||
.add("z", ColumnType.LONG)
|
||||
.build()
|
||||
);
|
||||
|
||||
/**
|
||||
* Basic use of EXTERN
|
||||
@ -262,7 +292,7 @@ public class IngestTableFunctionTest extends CalciteIngestionDmlTest
|
||||
new DefaultPasswordProvider("secret"),
|
||||
SystemFields.none(),
|
||||
ImmutableMap.of("Accept", "application/ndjson", "a", "b"),
|
||||
new HttpInputSourceConfig(null, null)
|
||||
new HttpInputSourceConfig(null, Sets.newHashSet("Accept", "a"))
|
||||
),
|
||||
new CsvInputFormat(ImmutableList.of("timestamp", "isRobot"), null, false, false, 0),
|
||||
RowSignature.builder()
|
||||
@ -549,21 +579,6 @@ public class IngestTableFunctionTest extends CalciteIngestionDmlTest
|
||||
.verify();
|
||||
}
|
||||
|
||||
protected final ExternalDataSource localDataSource = new ExternalDataSource(
|
||||
new LocalInputSource(
|
||||
null,
|
||||
null,
|
||||
Arrays.asList(new File("/tmp/foo.csv"), new File("/tmp/bar.csv")),
|
||||
SystemFields.none()
|
||||
),
|
||||
new CsvInputFormat(ImmutableList.of("x", "y", "z"), null, false, false, 0),
|
||||
RowSignature.builder()
|
||||
.add("x", ColumnType.STRING)
|
||||
.add("y", ColumnType.STRING)
|
||||
.add("z", ColumnType.LONG)
|
||||
.build()
|
||||
);
|
||||
|
||||
/**
|
||||
* Basic use of LOCALFILES
|
||||
*/
|
||||
@ -702,4 +717,66 @@ public class IngestTableFunctionTest extends CalciteIngestionDmlTest
|
||||
.expectLogicalPlanFrom("localExtern")
|
||||
.verify();
|
||||
}
|
||||
|
||||
protected static class ExportComponentSupplier extends IngestionDmlComponentSupplier
|
||||
{
|
||||
public ExportComponentSupplier(TempDirProducer tempFolderProducer)
|
||||
{
|
||||
super(tempFolderProducer);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void configureGuice(DruidInjectorBuilder builder)
|
||||
{
|
||||
builder.addModule(new DruidModule()
|
||||
{
|
||||
|
||||
// Clone of MSQExternalDataSourceModule since it is not
|
||||
// visible here.
|
||||
@Override
|
||||
public List<? extends Module> getJacksonModules()
|
||||
{
|
||||
return Collections.singletonList(
|
||||
new SimpleModule(getClass().getSimpleName())
|
||||
.registerSubtypes(ExternalDataSource.class)
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void configure(Binder binder)
|
||||
{
|
||||
// Adding the config to allow following 2 headers.
|
||||
binder.bind(HttpInputSourceConfig.class)
|
||||
.toInstance(new HttpInputSourceConfig(null, ImmutableSet.of("Accept", "a")));
|
||||
|
||||
}
|
||||
});
|
||||
|
||||
builder.addModule(new DruidModule()
|
||||
{
|
||||
|
||||
@Override
|
||||
public List<? extends Module> getJacksonModules()
|
||||
{
|
||||
// We want this module to bring input sources along for the ride.
|
||||
List<Module> modules = new ArrayList<>(new InputSourceModule().getJacksonModules());
|
||||
modules.add(new SimpleModule("test-module").registerSubtypes(TestFileInputSource.class));
|
||||
return modules;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void configure(Binder binder)
|
||||
{
|
||||
// Set up the EXTERN macro.
|
||||
SqlBindings.addOperatorConversion(binder, ExternalOperatorConversion.class);
|
||||
|
||||
// Enable the extended table functions for testing even though these
|
||||
// are not enabled in production in Druid 26.
|
||||
SqlBindings.addOperatorConversion(binder, HttpOperatorConversion.class);
|
||||
SqlBindings.addOperatorConversion(binder, InlineOperatorConversion.class);
|
||||
SqlBindings.addOperatorConversion(binder, LocalOperatorConversion.class);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user