Cleaner JSON for various input sources and formats. (#13064)

* Cleaner JSON for various input sources and formats.

Add JsonInclude to various properties, to avoid population of default
values in serialized JSON.

Also fixes a bug in OrcInputFormat: it was not writing binaryAsString,
so the property would be lost on serde.

* Additonal test cases.
This commit is contained in:
Gian Merlino 2022-09-12 10:29:31 -07:00 committed by GitHub
parent 80b97ac24d
commit c00ad28ecc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 250 additions and 32 deletions

View File

@ -19,6 +19,7 @@
package org.apache.druid.data.input.impl; package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.primitives.Ints; import com.google.common.primitives.Ints;
import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.FilenameUtils;
@ -83,12 +84,14 @@ public abstract class CloudObjectInputSource extends AbstractInputSource
} }
@JsonProperty @JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public List<URI> getUris() public List<URI> getUris()
{ {
return uris; return uris;
} }
@JsonProperty @JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public List<URI> getPrefixes() public List<URI> getPrefixes()
{ {
return prefixes; return prefixes;
@ -96,6 +99,7 @@ public abstract class CloudObjectInputSource extends AbstractInputSource
@Nullable @Nullable
@JsonProperty @JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public List<CloudObjectLocation> getObjects() public List<CloudObjectLocation> getObjects()
{ {
return objects; return objects;
@ -103,6 +107,7 @@ public abstract class CloudObjectInputSource extends AbstractInputSource
@Nullable @Nullable
@JsonProperty @JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public String getFilter() public String getFilter()
{ {
return filter; return filter;

View File

@ -19,6 +19,7 @@
package org.apache.druid.data.input.impl; package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
@ -92,24 +93,29 @@ public abstract class FlatTextInputFormat implements InputFormat
} }
@JsonProperty @JsonProperty
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public List<String> getColumns() public List<String> getColumns()
{ {
return columns; return columns;
} }
@Nullable
@JsonProperty @JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public String getListDelimiter() public String getListDelimiter()
{ {
return listDelimiter; return listDelimiter;
} }
@JsonProperty @JsonProperty
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
public boolean isFindColumnsFromHeader() public boolean isFindColumnsFromHeader()
{ {
return findColumnsFromHeader; return findColumnsFromHeader;
} }
@JsonProperty @JsonProperty
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
public int getSkipHeaderRows() public int getSkipHeaderRows()
{ {
return skipHeaderRows; return skipHeaderRows;

View File

@ -21,6 +21,7 @@ package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import org.apache.druid.data.input.AbstractInputSource; import org.apache.druid.data.input.AbstractInputSource;
@ -83,6 +84,7 @@ public class HttpInputSource extends AbstractInputSource implements SplittableIn
@Nullable @Nullable
@JsonProperty @JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public String getHttpAuthenticationUsername() public String getHttpAuthenticationUsername()
{ {
return httpAuthenticationUsername; return httpAuthenticationUsername;
@ -90,6 +92,7 @@ public class HttpInputSource extends AbstractInputSource implements SplittableIn
@Nullable @Nullable
@JsonProperty("httpAuthenticationPassword") @JsonProperty("httpAuthenticationPassword")
@JsonInclude(JsonInclude.Include.NON_NULL)
public PasswordProvider getHttpAuthenticationPasswordProvider() public PasswordProvider getHttpAuthenticationPasswordProvider()
{ {
return httpAuthenticationPasswordProvider; return httpAuthenticationPasswordProvider;

View File

@ -20,6 +20,7 @@
package org.apache.druid.data.input.impl; package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonParser.Feature; import com.fasterxml.jackson.core.JsonParser.Feature;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
@ -87,12 +88,13 @@ public class JsonInputFormat extends NestedInputFormat
} }
@JsonProperty @JsonProperty
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public Map<String, Boolean> getFeatureSpec() public Map<String, Boolean> getFeatureSpec()
{ {
return featureSpec; return featureSpec;
} }
@JsonProperty @JsonProperty // No @JsonInclude, since default is variable, so we can't assume false is default
public boolean isKeepNullColumns() public boolean isKeepNullColumns()
{ {
return keepNullColumns; return keepNullColumns;

View File

@ -20,6 +20,7 @@
package org.apache.druid.data.input.impl; package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
@ -86,6 +87,7 @@ public class LocalInputSource extends AbstractInputSource implements SplittableI
@Nullable @Nullable
@JsonProperty @JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public File getBaseDir() public File getBaseDir()
{ {
return baseDir; return baseDir;
@ -93,12 +95,14 @@ public class LocalInputSource extends AbstractInputSource implements SplittableI
@Nullable @Nullable
@JsonProperty @JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public String getFilter() public String getFilter()
{ {
return filter; return filter;
} }
@JsonProperty @JsonProperty
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public List<File> getFiles() public List<File> getFiles()
{ {
return files; return files;

View File

@ -19,6 +19,7 @@
package org.apache.druid.data.input.impl; package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.InputFormat;
import org.apache.druid.java.util.common.parsers.JSONPathSpec; import org.apache.druid.java.util.common.parsers.JSONPathSpec;
@ -42,6 +43,7 @@ public abstract class NestedInputFormat implements InputFormat
@Nullable @Nullable
@JsonProperty("flattenSpec") @JsonProperty("flattenSpec")
@JsonInclude(JsonInclude.Include.NON_NULL)
public JSONPathSpec getFlattenSpec() public JSONPathSpec getFlattenSpec()
{ {
return flattenSpec; return flattenSpec;

View File

@ -21,6 +21,7 @@ package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Supplier; import com.google.common.base.Supplier;
import com.google.common.base.Suppliers; import com.google.common.base.Suppliers;
@ -63,6 +64,7 @@ public class RegexInputFormat implements InputFormat
@Nullable @Nullable
@JsonProperty @JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public String getListDelimiter() public String getListDelimiter()
{ {
return listDelimiter; return listDelimiter;
@ -70,6 +72,7 @@ public class RegexInputFormat implements InputFormat
@Nullable @Nullable
@JsonProperty @JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public List<String> getColumns() public List<String> getColumns()
{ {
return columns; return columns;

View File

@ -21,6 +21,7 @@ package org.apache.druid.data.input.avro;
import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.avro.Schema; import org.apache.avro.Schema;
@ -79,19 +80,23 @@ public class AvroOCFInputFormat extends NestedInputFormat
return false; return false;
} }
@Nullable
@JsonProperty @JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public Map<String, Object> getSchema() public Map<String, Object> getSchema()
{ {
return schema; return schema;
} }
@JsonProperty @JsonProperty
public Boolean getBinaryAsString() @JsonInclude(JsonInclude.Include.NON_DEFAULT)
public boolean getBinaryAsString()
{ {
return binaryAsString; return binaryAsString;
} }
@JsonProperty @JsonProperty
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
public Boolean isExtractUnionsByType() public Boolean isExtractUnionsByType()
{ {
return extractUnionsByType; return extractUnionsByType;

View File

@ -218,6 +218,11 @@
<artifactId>junit</artifactId> <artifactId>junit</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>nl.jqno.equalsverifier</groupId>
<artifactId>equalsverifier</artifactId>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.apache.druid</groupId> <groupId>org.apache.druid</groupId>
<artifactId>druid-core</artifactId> <artifactId>druid-core</artifactId>
@ -225,6 +230,13 @@
<type>test-jar</type> <type>test-jar</type>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.apache.druid</groupId>
<artifactId>druid-processing</artifactId>
<version>${project.parent.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies> </dependencies>
<profiles> <profiles>
<profile> <profile>

View File

@ -21,6 +21,7 @@ package org.apache.druid.data.input.orc;
import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.druid.data.input.InputEntity; import org.apache.druid.data.input.InputEntity;
import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputEntityReader;
@ -49,7 +50,7 @@ public class OrcInputFormat extends NestedInputFormat
) )
{ {
super(flattenSpec); super(flattenSpec);
this.binaryAsString = binaryAsString == null ? false : binaryAsString; this.binaryAsString = binaryAsString != null && binaryAsString;
this.conf = conf; this.conf = conf;
} }
@ -79,6 +80,13 @@ public class OrcInputFormat extends NestedInputFormat
return false; return false;
} }
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
public boolean getBinaryAsString()
{
return binaryAsString;
}
@Override @Override
public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory) public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory)
{ {

View File

@ -0,0 +1,79 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.data.input.orc;
import com.fasterxml.jackson.databind.InjectableValues;
import com.fasterxml.jackson.databind.ObjectMapper;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.data.input.InputFormat;
import org.apache.druid.java.util.common.parsers.JSONPathSpec;
import org.apache.druid.segment.TestHelper;
import org.apache.hadoop.conf.Configuration;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.util.Collections;
public class OrcInputFormatTest
{
private ObjectMapper mapper;
@Before
public void setUp()
{
mapper =
TestHelper.makeJsonMapper()
.registerModules(new OrcExtensionsModule().getJacksonModules())
.setInjectableValues(new InjectableValues.Std().addValue(Configuration.class, null));
}
@Test
public void testSerdeDefault() throws Exception
{
final OrcInputFormat config = new OrcInputFormat(null, null, null);
Assert.assertEquals(
config,
mapper.readValue(mapper.writeValueAsString(config), InputFormat.class)
);
}
@Test
public void testSerdeNonDefault() throws Exception
{
final OrcInputFormat config = new OrcInputFormat(new JSONPathSpec(true, Collections.emptyList()), true, null);
Assert.assertEquals(
config,
mapper.readValue(mapper.writeValueAsString(config), InputFormat.class)
);
}
@Test
public void testEquals()
{
EqualsVerifier.forClass(OrcInputFormat.class)
.withPrefabValues(Configuration.class, new Configuration(), new Configuration())
.usingGetClass()
.verify();
}
}

View File

@ -21,6 +21,7 @@ package org.apache.druid.data.input.parquet;
import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.druid.data.input.InputEntity; import org.apache.druid.data.input.InputEntity;
import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputEntityReader;
@ -74,6 +75,7 @@ public class ParquetInputFormat extends NestedInputFormat
} }
@JsonProperty @JsonProperty
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
public boolean getBinaryAsString() public boolean getBinaryAsString()
{ {
return binaryAsString; return binaryAsString;

View File

@ -168,6 +168,11 @@
<artifactId>hamcrest-core</artifactId> <artifactId>hamcrest-core</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>nl.jqno.equalsverifier</groupId>
<artifactId>equalsverifier</artifactId>
<scope>test</scope>
</dependency>
</dependencies> </dependencies>
</project> </project>

View File

@ -30,6 +30,7 @@ import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder; import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder;
import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
@ -77,7 +78,6 @@ public class S3InputSource extends CloudObjectInputSource
private final AWSProxyConfig awsProxyConfig; private final AWSProxyConfig awsProxyConfig;
private final AWSClientConfig awsClientConfig; private final AWSClientConfig awsClientConfig;
private final AWSEndpointConfig awsEndpointConfig; private final AWSEndpointConfig awsEndpointConfig;
private final AWSCredentialsProvider awsCredentialsProvider;
private int maxRetries; private int maxRetries;
/** /**
@ -104,6 +104,7 @@ public class S3InputSource extends CloudObjectInputSource
@JacksonInject ServerSideEncryptingAmazonS3 s3Client, @JacksonInject ServerSideEncryptingAmazonS3 s3Client,
@JacksonInject ServerSideEncryptingAmazonS3.Builder s3ClientBuilder, @JacksonInject ServerSideEncryptingAmazonS3.Builder s3ClientBuilder,
@JacksonInject S3InputDataConfig inputDataConfig, @JacksonInject S3InputDataConfig inputDataConfig,
@JacksonInject AWSCredentialsProvider awsCredentialsProvider,
@JsonProperty("uris") @Nullable List<URI> uris, @JsonProperty("uris") @Nullable List<URI> uris,
@JsonProperty("prefixes") @Nullable List<URI> prefixes, @JsonProperty("prefixes") @Nullable List<URI> prefixes,
@JsonProperty("objects") @Nullable List<CloudObjectLocation> objects, @JsonProperty("objects") @Nullable List<CloudObjectLocation> objects,
@ -111,8 +112,7 @@ public class S3InputSource extends CloudObjectInputSource
@JsonProperty("properties") @Nullable S3InputSourceConfig s3InputSourceConfig, @JsonProperty("properties") @Nullable S3InputSourceConfig s3InputSourceConfig,
@JsonProperty("proxyConfig") @Nullable AWSProxyConfig awsProxyConfig, @JsonProperty("proxyConfig") @Nullable AWSProxyConfig awsProxyConfig,
@JsonProperty("endpointConfig") @Nullable AWSEndpointConfig awsEndpointConfig, @JsonProperty("endpointConfig") @Nullable AWSEndpointConfig awsEndpointConfig,
@JsonProperty("clientConfig") @Nullable AWSClientConfig awsClientConfig, @JsonProperty("clientConfig") @Nullable AWSClientConfig awsClientConfig
@JacksonInject AWSCredentialsProvider awsCredentialsProvider
) )
{ {
super(S3StorageDruidModule.SCHEME, uris, prefixes, objects, filter); super(S3StorageDruidModule.SCHEME, uris, prefixes, objects, filter);
@ -174,11 +174,10 @@ public class S3InputSource extends CloudObjectInputSource
} }
); );
this.maxRetries = RetryUtils.DEFAULT_MAX_TRIES; this.maxRetries = RetryUtils.DEFAULT_MAX_TRIES;
this.awsCredentialsProvider = awsCredentialsProvider;
} }
@VisibleForTesting @VisibleForTesting
public S3InputSource( S3InputSource(
ServerSideEncryptingAmazonS3 s3Client, ServerSideEncryptingAmazonS3 s3Client,
ServerSideEncryptingAmazonS3.Builder s3ClientBuilder, ServerSideEncryptingAmazonS3.Builder s3ClientBuilder,
S3InputDataConfig inputDataConfig, S3InputDataConfig inputDataConfig,
@ -192,18 +191,19 @@ public class S3InputSource extends CloudObjectInputSource
AWSClientConfig awsClientConfig AWSClientConfig awsClientConfig
) )
{ {
this(s3Client, this(
s3ClientBuilder, s3Client,
inputDataConfig, s3ClientBuilder,
uris, inputDataConfig,
prefixes, null,
objects, uris,
filter, prefixes,
s3InputSourceConfig, objects,
awsProxyConfig, filter,
awsEndpointConfig, s3InputSourceConfig,
awsClientConfig, awsProxyConfig,
null awsEndpointConfig,
awsClientConfig
); );
} }
@ -227,6 +227,7 @@ public class S3InputSource extends CloudObjectInputSource
s3Client, s3Client,
s3ClientBuilder, s3ClientBuilder,
inputDataConfig, inputDataConfig,
null,
uris, uris,
prefixes, prefixes,
objects, objects,
@ -234,8 +235,7 @@ public class S3InputSource extends CloudObjectInputSource
s3InputSourceConfig, s3InputSourceConfig,
awsProxyConfig, awsProxyConfig,
awsEndpointConfig, awsEndpointConfig,
awsClientConfig, awsClientConfig
null
); );
this.maxRetries = maxRetries; this.maxRetries = maxRetries;
} }
@ -278,6 +278,7 @@ public class S3InputSource extends CloudObjectInputSource
@Nullable @Nullable
@JsonProperty("properties") @JsonProperty("properties")
@JsonInclude(JsonInclude.Include.NON_NULL)
public S3InputSourceConfig getS3InputSourceConfig() public S3InputSourceConfig getS3InputSourceConfig()
{ {
return s3InputSourceConfig; return s3InputSourceConfig;
@ -285,6 +286,7 @@ public class S3InputSource extends CloudObjectInputSource
@Nullable @Nullable
@JsonProperty("proxyConfig") @JsonProperty("proxyConfig")
@JsonInclude(JsonInclude.Include.NON_NULL)
public AWSProxyConfig getAwsProxyConfig() public AWSProxyConfig getAwsProxyConfig()
{ {
return awsProxyConfig; return awsProxyConfig;
@ -292,6 +294,7 @@ public class S3InputSource extends CloudObjectInputSource
@Nullable @Nullable
@JsonProperty("clientConfig") @JsonProperty("clientConfig")
@JsonInclude(JsonInclude.Include.NON_NULL)
public AWSClientConfig getAwsClientConfig() public AWSClientConfig getAwsClientConfig()
{ {
return awsClientConfig; return awsClientConfig;
@ -299,6 +302,7 @@ public class S3InputSource extends CloudObjectInputSource
@Nullable @Nullable
@JsonProperty("endpointConfig") @JsonProperty("endpointConfig")
@JsonInclude(JsonInclude.Include.NON_NULL)
public AWSEndpointConfig getAwsEndpointConfig() public AWSEndpointConfig getAwsEndpointConfig()
{ {
return awsEndpointConfig; return awsEndpointConfig;
@ -334,13 +338,13 @@ public class S3InputSource extends CloudObjectInputSource
inputDataConfig, inputDataConfig,
null, null,
null, null,
null,
split.get(), split.get(),
getFilter(), getFilter(),
getS3InputSourceConfig(), getS3InputSourceConfig(),
getAwsProxyConfig(), getAwsProxyConfig(),
getAwsEndpointConfig(), getAwsEndpointConfig(),
getAwsClientConfig(), getAwsClientConfig()
awsCredentialsProvider
); );
} }

View File

@ -21,6 +21,7 @@ package org.apache.druid.data.input.s3;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import org.apache.druid.metadata.PasswordProvider; import org.apache.druid.metadata.PasswordProvider;
@ -34,14 +35,10 @@ import java.util.Objects;
*/ */
public class S3InputSourceConfig public class S3InputSourceConfig
{ {
@JsonProperty private final String assumeRoleArn;
private String assumeRoleArn; private final String assumeRoleExternalId;
@JsonProperty private final PasswordProvider accessKeyId;
private String assumeRoleExternalId; private final PasswordProvider secretAccessKey;
@JsonProperty
private PasswordProvider accessKeyId;
@JsonProperty
private PasswordProvider secretAccessKey;
@JsonCreator @JsonCreator
public S3InputSourceConfig( public S3InputSourceConfig(
@ -56,28 +53,39 @@ public class S3InputSourceConfig
if (accessKeyId != null || secretAccessKey != null) { if (accessKeyId != null || secretAccessKey != null) {
this.accessKeyId = Preconditions.checkNotNull(accessKeyId, "accessKeyId cannot be null if secretAccessKey is given"); this.accessKeyId = Preconditions.checkNotNull(accessKeyId, "accessKeyId cannot be null if secretAccessKey is given");
this.secretAccessKey = Preconditions.checkNotNull(secretAccessKey, "secretAccessKey cannot be null if accessKeyId is given"); this.secretAccessKey = Preconditions.checkNotNull(secretAccessKey, "secretAccessKey cannot be null if accessKeyId is given");
} else {
this.accessKeyId = null;
this.secretAccessKey = null;
} }
} }
@Nullable @Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public String getAssumeRoleArn() public String getAssumeRoleArn()
{ {
return assumeRoleArn; return assumeRoleArn;
} }
@Nullable @Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public String getAssumeRoleExternalId() public String getAssumeRoleExternalId()
{ {
return assumeRoleExternalId; return assumeRoleExternalId;
} }
@Nullable @Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public PasswordProvider getAccessKeyId() public PasswordProvider getAccessKeyId()
{ {
return accessKeyId; return accessKeyId;
} }
@Nullable @Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public PasswordProvider getSecretAccessKey() public PasswordProvider getSecretAccessKey()
{ {
return secretAccessKey; return secretAccessKey;

View File

@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.data.input.s3;
import com.fasterxml.jackson.databind.ObjectMapper;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.metadata.DefaultPasswordProvider;
import org.apache.druid.segment.TestHelper;
import org.junit.Assert;
import org.junit.Test;
public class S3InputSourceConfigTest
{
@Test
public void testSerdeAccessSecretKey() throws Exception
{
final ObjectMapper mapper = TestHelper.makeJsonMapper();
final S3InputSourceConfig config = new S3InputSourceConfig(
new DefaultPasswordProvider("the-access-key"),
new DefaultPasswordProvider("the-secret-key"),
null,
null
);
Assert.assertEquals(
config,
mapper.readValue(mapper.writeValueAsString(config), S3InputSourceConfig.class)
);
}
@Test
public void testSerdeAssumeRole() throws Exception
{
final ObjectMapper mapper = TestHelper.makeJsonMapper();
final S3InputSourceConfig config = new S3InputSourceConfig(
null,
null,
"the-role-arn",
"the-role-external-id"
);
Assert.assertEquals(
config,
mapper.readValue(mapper.writeValueAsString(config), S3InputSourceConfig.class)
);
}
@Test
public void testEquals()
{
EqualsVerifier.forClass(S3InputSourceConfig.class).usingGetClass().verify();
}
}