Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cleaner JSON for various input sources and formats. (#13064) 

* Cleaner JSON for various input sources and formats.

Add JsonInclude to various properties, to avoid population of default
values in serialized JSON.

Also fixes a bug in OrcInputFormat: it was not writing binaryAsString,
so the property would be lost on serde.

* Additonal test cases.
This commit is contained in:
Gian Merlino 2022-09-12 10:29:31 -07:00 committed by GitHub
parent 80b97ac24d
commit c00ad28ecc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 250 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
core/src/main/java/org/apache/druid/data/input/impl/CloudObjectInputSource.java
View File

@ -19,6 +19,7 @@
package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.primitives.Ints;
import org.apache.commons.io.FilenameUtils;
@ -83,12 +84,14 @@ public abstract class CloudObjectInputSource extends AbstractInputSource
}
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public List<URI> getUris()
{
return uris;
}
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public List<URI> getPrefixes()
{
return prefixes;
@ -96,6 +99,7 @@ public abstract class CloudObjectInputSource extends AbstractInputSource
@Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public List<CloudObjectLocation> getObjects()
{
return objects;
@ -103,6 +107,7 @@ public abstract class CloudObjectInputSource extends AbstractInputSource
@Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public String getFilter()
{
return filter;

6
core/src/main/java/org/apache/druid/data/input/impl/FlatTextInputFormat.java
View File

@ -19,6 +19,7 @@
package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@ -92,24 +93,29 @@ public abstract class FlatTextInputFormat implements InputFormat
}
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public List<String> getColumns()
{
return columns;
}
@Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public String getListDelimiter()
{
return listDelimiter;
}
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
public boolean isFindColumnsFromHeader()
{
return findColumnsFromHeader;
}
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
public int getSkipHeaderRows()
{
return skipHeaderRows;

3
core/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java
View File

@ -21,6 +21,7 @@ package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import org.apache.druid.data.input.AbstractInputSource;
@ -83,6 +84,7 @@ public class HttpInputSource extends AbstractInputSource implements SplittableIn
@Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public String getHttpAuthenticationUsername()
{
return httpAuthenticationUsername;
@ -90,6 +92,7 @@ public class HttpInputSource extends AbstractInputSource implements SplittableIn
@Nullable
@JsonProperty("httpAuthenticationPassword")
@JsonInclude(JsonInclude.Include.NON_NULL)
public PasswordProvider getHttpAuthenticationPasswordProvider()
{
return httpAuthenticationPasswordProvider;

4
core/src/main/java/org/apache/druid/data/input/impl/JsonInputFormat.java
View File

@ -20,6 +20,7 @@
package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonParser.Feature;
import com.fasterxml.jackson.databind.ObjectMapper;
@ -87,12 +88,13 @@ public class JsonInputFormat extends NestedInputFormat
}
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public Map<String, Boolean> getFeatureSpec()
{
return featureSpec;
}
@JsonProperty
@JsonProperty // No @JsonInclude, since default is variable, so we can't assume false is default
public boolean isKeepNullColumns()
{
return keepNullColumns;

4
core/src/main/java/org/apache/druid/data/input/impl/LocalInputSource.java
View File

@ -20,6 +20,7 @@
package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@ -86,6 +87,7 @@ public class LocalInputSource extends AbstractInputSource implements SplittableI
@Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public File getBaseDir()
{
return baseDir;
@ -93,12 +95,14 @@ public class LocalInputSource extends AbstractInputSource implements SplittableI
@Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public String getFilter()
{
return filter;
}
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public List<File> getFiles()
{
return files;

2
core/src/main/java/org/apache/druid/data/input/impl/NestedInputFormat.java
View File

@ -19,6 +19,7 @@
package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.druid.data.input.InputFormat;
import org.apache.druid.java.util.common.parsers.JSONPathSpec;
@ -42,6 +43,7 @@ public abstract class NestedInputFormat implements InputFormat
@Nullable
@JsonProperty("flattenSpec")
@JsonInclude(JsonInclude.Include.NON_NULL)
public JSONPathSpec getFlattenSpec()
{
return flattenSpec;

3
core/src/main/java/org/apache/druid/data/input/impl/RegexInputFormat.java
View File

@ -21,6 +21,7 @@ package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
@ -63,6 +64,7 @@ public class RegexInputFormat implements InputFormat
@Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public String getListDelimiter()
{
return listDelimiter;
@ -70,6 +72,7 @@ public class RegexInputFormat implements InputFormat
@Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public List<String> getColumns()
{
return columns;

7
extensions-core/avro-extensions/src/main/java/org/apache/druid/data/input/avro/AvroOCFInputFormat.java
View File

@ -21,6 +21,7 @@ package org.apache.druid.data.input.avro;
import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.avro.Schema;
@ -79,19 +80,23 @@ public class AvroOCFInputFormat extends NestedInputFormat
return false;
}
@Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public Map<String, Object> getSchema()
{
return schema;
}
@JsonProperty
public Boolean getBinaryAsString()
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
public boolean getBinaryAsString()
{
return binaryAsString;
}
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
public Boolean isExtractUnionsByType()
{
return extractUnionsByType;

12
extensions-core/orc-extensions/pom.xml
View File

@ -218,6 +218,11 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>nl.jqno.equalsverifier</groupId>
<artifactId>equalsverifier</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.druid</groupId>
<artifactId>druid-core</artifactId>
@ -225,6 +230,13 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.druid</groupId>
<artifactId>druid-processing</artifactId>
<version>${project.parent.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
<profiles>
<profile>

10
extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcInputFormat.java
View File

@ -21,6 +21,7 @@ package org.apache.druid.data.input.orc;
import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.druid.data.input.InputEntity;
import org.apache.druid.data.input.InputEntityReader;
@ -49,7 +50,7 @@ public class OrcInputFormat extends NestedInputFormat
)
{
super(flattenSpec);
this.binaryAsString = binaryAsString == null ? false : binaryAsString;
this.binaryAsString = binaryAsString != null && binaryAsString;
this.conf = conf;
}
@ -79,6 +80,13 @@ public class OrcInputFormat extends NestedInputFormat
return false;
}
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
public boolean getBinaryAsString()
{
return binaryAsString;
}
@Override
public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory)
{

79
extensions-core/orc-extensions/src/test/java/org/apache/druid/data/input/orc/OrcInputFormatTest.java Normal file
View File

@ -0,0 +1,79 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.data.input.orc;
import com.fasterxml.jackson.databind.InjectableValues;
import com.fasterxml.jackson.databind.ObjectMapper;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.data.input.InputFormat;
import org.apache.druid.java.util.common.parsers.JSONPathSpec;
import org.apache.druid.segment.TestHelper;
import org.apache.hadoop.conf.Configuration;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.util.Collections;
public class OrcInputFormatTest
{
private ObjectMapper mapper;
@Before
public void setUp()
{
mapper =
TestHelper.makeJsonMapper()
.registerModules(new OrcExtensionsModule().getJacksonModules())
.setInjectableValues(new InjectableValues.Std().addValue(Configuration.class, null));
}
@Test
public void testSerdeDefault() throws Exception
{
final OrcInputFormat config = new OrcInputFormat(null, null, null);
Assert.assertEquals(
config,
mapper.readValue(mapper.writeValueAsString(config), InputFormat.class)
);
}
@Test
public void testSerdeNonDefault() throws Exception
{
final OrcInputFormat config = new OrcInputFormat(new JSONPathSpec(true, Collections.emptyList()), true, null);
Assert.assertEquals(
config,
mapper.readValue(mapper.writeValueAsString(config), InputFormat.class)
);
}
@Test
public void testEquals()
{
EqualsVerifier.forClass(OrcInputFormat.class)
.withPrefabValues(Configuration.class, new Configuration(), new Configuration())
.usingGetClass()
.verify();
}
}

2
extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/ParquetInputFormat.java
View File

@ -21,6 +21,7 @@ package org.apache.druid.data.input.parquet;
import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.druid.data.input.InputEntity;
import org.apache.druid.data.input.InputEntityReader;
@ -74,6 +75,7 @@ public class ParquetInputFormat extends NestedInputFormat
}
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
public boolean getBinaryAsString()
{
return binaryAsString;

5
extensions-core/s3-extensions/pom.xml
View File

@ -168,6 +168,11 @@
<artifactId>hamcrest-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>nl.jqno.equalsverifier</groupId>
<artifactId>equalsverifier</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

46
extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSource.java
View File

@ -30,6 +30,7 @@ import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder;
import com.fasterxml.jackson.annotation.JacksonInject;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@ -77,7 +78,6 @@ public class S3InputSource extends CloudObjectInputSource
private final AWSProxyConfig awsProxyConfig;
private final AWSClientConfig awsClientConfig;
private final AWSEndpointConfig awsEndpointConfig;
private final AWSCredentialsProvider awsCredentialsProvider;
private int maxRetries;
/**
@ -104,6 +104,7 @@ public class S3InputSource extends CloudObjectInputSource
@JacksonInject ServerSideEncryptingAmazonS3 s3Client,
@JacksonInject ServerSideEncryptingAmazonS3.Builder s3ClientBuilder,
@JacksonInject S3InputDataConfig inputDataConfig,
@JacksonInject AWSCredentialsProvider awsCredentialsProvider,
@JsonProperty("uris") @Nullable List<URI> uris,
@JsonProperty("prefixes") @Nullable List<URI> prefixes,
@JsonProperty("objects") @Nullable List<CloudObjectLocation> objects,
@ -111,8 +112,7 @@ public class S3InputSource extends CloudObjectInputSource
@JsonProperty("properties") @Nullable S3InputSourceConfig s3InputSourceConfig,
@JsonProperty("proxyConfig") @Nullable AWSProxyConfig awsProxyConfig,
@JsonProperty("endpointConfig") @Nullable AWSEndpointConfig awsEndpointConfig,
@JsonProperty("clientConfig") @Nullable AWSClientConfig awsClientConfig,
@JacksonInject AWSCredentialsProvider awsCredentialsProvider
@JsonProperty("clientConfig") @Nullable AWSClientConfig awsClientConfig
)
{
super(S3StorageDruidModule.SCHEME, uris, prefixes, objects, filter);
@ -174,11 +174,10 @@ public class S3InputSource extends CloudObjectInputSource
}
);
this.maxRetries = RetryUtils.DEFAULT_MAX_TRIES;
this.awsCredentialsProvider = awsCredentialsProvider;
}
@VisibleForTesting
public S3InputSource(
S3InputSource(
ServerSideEncryptingAmazonS3 s3Client,
ServerSideEncryptingAmazonS3.Builder s3ClientBuilder,
S3InputDataConfig inputDataConfig,
@ -192,18 +191,19 @@ public class S3InputSource extends CloudObjectInputSource
AWSClientConfig awsClientConfig
)
{
this(s3Client,
s3ClientBuilder,
inputDataConfig,
uris,
prefixes,
objects,
filter,
s3InputSourceConfig,
awsProxyConfig,
awsEndpointConfig,
awsClientConfig,
null
this(
s3Client,
s3ClientBuilder,
inputDataConfig,
null,
uris,
prefixes,
objects,
filter,
s3InputSourceConfig,
awsProxyConfig,
awsEndpointConfig,
awsClientConfig
);
}
@ -227,6 +227,7 @@ public class S3InputSource extends CloudObjectInputSource
s3Client,
s3ClientBuilder,
inputDataConfig,
null,
uris,
prefixes,
objects,
@ -234,8 +235,7 @@ public class S3InputSource extends CloudObjectInputSource
s3InputSourceConfig,
awsProxyConfig,
awsEndpointConfig,
awsClientConfig,
null
awsClientConfig
);
this.maxRetries = maxRetries;
}
@ -278,6 +278,7 @@ public class S3InputSource extends CloudObjectInputSource
@Nullable
@JsonProperty("properties")
@JsonInclude(JsonInclude.Include.NON_NULL)
public S3InputSourceConfig getS3InputSourceConfig()
{
return s3InputSourceConfig;
@ -285,6 +286,7 @@ public class S3InputSource extends CloudObjectInputSource
@Nullable
@JsonProperty("proxyConfig")
@JsonInclude(JsonInclude.Include.NON_NULL)
public AWSProxyConfig getAwsProxyConfig()
{
return awsProxyConfig;
@ -292,6 +294,7 @@ public class S3InputSource extends CloudObjectInputSource
@Nullable
@JsonProperty("clientConfig")
@JsonInclude(JsonInclude.Include.NON_NULL)
public AWSClientConfig getAwsClientConfig()
{
return awsClientConfig;
@ -299,6 +302,7 @@ public class S3InputSource extends CloudObjectInputSource
@Nullable
@JsonProperty("endpointConfig")
@JsonInclude(JsonInclude.Include.NON_NULL)
public AWSEndpointConfig getAwsEndpointConfig()
{
return awsEndpointConfig;
@ -334,13 +338,13 @@ public class S3InputSource extends CloudObjectInputSource
inputDataConfig,
null,
null,
null,
split.get(),
getFilter(),
getS3InputSourceConfig(),
getAwsProxyConfig(),
getAwsEndpointConfig(),
getAwsClientConfig(),
awsCredentialsProvider
getAwsClientConfig()
);
}

24
extensions-core/s3-extensions/src/main/java/org/apache/druid/data/input/s3/S3InputSourceConfig.java
View File

@ -21,6 +21,7 @@ package org.apache.druid.data.input.s3;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import org.apache.druid.metadata.PasswordProvider;
@ -34,14 +35,10 @@ import java.util.Objects;
*/
public class S3InputSourceConfig
{
@JsonProperty
private String assumeRoleArn;
@JsonProperty
private String assumeRoleExternalId;
@JsonProperty
private PasswordProvider accessKeyId;
@JsonProperty
private PasswordProvider secretAccessKey;
private final String assumeRoleArn;
private final String assumeRoleExternalId;
private final PasswordProvider accessKeyId;
private final PasswordProvider secretAccessKey;
@JsonCreator
public S3InputSourceConfig(
@ -56,28 +53,39 @@ public class S3InputSourceConfig
if (accessKeyId != null || secretAccessKey != null) {
this.accessKeyId = Preconditions.checkNotNull(accessKeyId, "accessKeyId cannot be null if secretAccessKey is given");
this.secretAccessKey = Preconditions.checkNotNull(secretAccessKey, "secretAccessKey cannot be null if accessKeyId is given");
} else {
this.accessKeyId = null;
this.secretAccessKey = null;
}
}
@Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public String getAssumeRoleArn()
{
return assumeRoleArn;
}
@Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public String getAssumeRoleExternalId()
{
return assumeRoleExternalId;
}
@Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public PasswordProvider getAccessKeyId()
{
return accessKeyId;
}
@Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public PasswordProvider getSecretAccessKey()
{
return secretAccessKey;

70
extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceConfigTest.java Normal file
View File

@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.data.input.s3;
import com.fasterxml.jackson.databind.ObjectMapper;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.metadata.DefaultPasswordProvider;
import org.apache.druid.segment.TestHelper;
import org.junit.Assert;
import org.junit.Test;
public class S3InputSourceConfigTest
{
@Test
public void testSerdeAccessSecretKey() throws Exception
{
final ObjectMapper mapper = TestHelper.makeJsonMapper();
final S3InputSourceConfig config = new S3InputSourceConfig(
new DefaultPasswordProvider("the-access-key"),
new DefaultPasswordProvider("the-secret-key"),
null,
null
);
Assert.assertEquals(
config,
mapper.readValue(mapper.writeValueAsString(config), S3InputSourceConfig.class)
);
}
@Test
public void testSerdeAssumeRole() throws Exception
{
final ObjectMapper mapper = TestHelper.makeJsonMapper();
final S3InputSourceConfig config = new S3InputSourceConfig(
null,
null,
"the-role-arn",
"the-role-external-id"
);
Assert.assertEquals(
config,
mapper.readValue(mapper.writeValueAsString(config), S3InputSourceConfig.class)
);
}
@Test
public void testEquals()
{
EqualsVerifier.forClass(S3InputSourceConfig.class).usingGetClass().verify();
}
}