mirror of https://github.com/apache/druid.git
Cleaner JSON for various input sources and formats. (#13064)
* Cleaner JSON for various input sources and formats. Add JsonInclude to various properties, to avoid population of default values in serialized JSON. Also fixes a bug in OrcInputFormat: it was not writing binaryAsString, so the property would be lost on serde. * Additonal test cases.
This commit is contained in:
parent
80b97ac24d
commit
c00ad28ecc
|
@ -19,6 +19,7 @@
|
||||||
|
|
||||||
package org.apache.druid.data.input.impl;
|
package org.apache.druid.data.input.impl;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import com.google.common.primitives.Ints;
|
import com.google.common.primitives.Ints;
|
||||||
import org.apache.commons.io.FilenameUtils;
|
import org.apache.commons.io.FilenameUtils;
|
||||||
|
@ -83,12 +84,14 @@ public abstract class CloudObjectInputSource extends AbstractInputSource
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public List<URI> getUris()
|
public List<URI> getUris()
|
||||||
{
|
{
|
||||||
return uris;
|
return uris;
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public List<URI> getPrefixes()
|
public List<URI> getPrefixes()
|
||||||
{
|
{
|
||||||
return prefixes;
|
return prefixes;
|
||||||
|
@ -96,6 +99,7 @@ public abstract class CloudObjectInputSource extends AbstractInputSource
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public List<CloudObjectLocation> getObjects()
|
public List<CloudObjectLocation> getObjects()
|
||||||
{
|
{
|
||||||
return objects;
|
return objects;
|
||||||
|
@ -103,6 +107,7 @@ public abstract class CloudObjectInputSource extends AbstractInputSource
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public String getFilter()
|
public String getFilter()
|
||||||
{
|
{
|
||||||
return filter;
|
return filter;
|
||||||
|
|
|
@ -19,6 +19,7 @@
|
||||||
|
|
||||||
package org.apache.druid.data.input.impl;
|
package org.apache.druid.data.input.impl;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import com.google.common.base.Preconditions;
|
import com.google.common.base.Preconditions;
|
||||||
import com.google.common.collect.ImmutableList;
|
import com.google.common.collect.ImmutableList;
|
||||||
|
@ -92,24 +93,29 @@ public abstract class FlatTextInputFormat implements InputFormat
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_EMPTY)
|
||||||
public List<String> getColumns()
|
public List<String> getColumns()
|
||||||
{
|
{
|
||||||
return columns;
|
return columns;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Nullable
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public String getListDelimiter()
|
public String getListDelimiter()
|
||||||
{
|
{
|
||||||
return listDelimiter;
|
return listDelimiter;
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
|
||||||
public boolean isFindColumnsFromHeader()
|
public boolean isFindColumnsFromHeader()
|
||||||
{
|
{
|
||||||
return findColumnsFromHeader;
|
return findColumnsFromHeader;
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
|
||||||
public int getSkipHeaderRows()
|
public int getSkipHeaderRows()
|
||||||
{
|
{
|
||||||
return skipHeaderRows;
|
return skipHeaderRows;
|
||||||
|
|
|
@ -21,6 +21,7 @@ package org.apache.druid.data.input.impl;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JacksonInject;
|
import com.fasterxml.jackson.annotation.JacksonInject;
|
||||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import com.google.common.base.Preconditions;
|
import com.google.common.base.Preconditions;
|
||||||
import org.apache.druid.data.input.AbstractInputSource;
|
import org.apache.druid.data.input.AbstractInputSource;
|
||||||
|
@ -83,6 +84,7 @@ public class HttpInputSource extends AbstractInputSource implements SplittableIn
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public String getHttpAuthenticationUsername()
|
public String getHttpAuthenticationUsername()
|
||||||
{
|
{
|
||||||
return httpAuthenticationUsername;
|
return httpAuthenticationUsername;
|
||||||
|
@ -90,6 +92,7 @@ public class HttpInputSource extends AbstractInputSource implements SplittableIn
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
@JsonProperty("httpAuthenticationPassword")
|
@JsonProperty("httpAuthenticationPassword")
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public PasswordProvider getHttpAuthenticationPasswordProvider()
|
public PasswordProvider getHttpAuthenticationPasswordProvider()
|
||||||
{
|
{
|
||||||
return httpAuthenticationPasswordProvider;
|
return httpAuthenticationPasswordProvider;
|
||||||
|
|
|
@ -20,6 +20,7 @@
|
||||||
package org.apache.druid.data.input.impl;
|
package org.apache.druid.data.input.impl;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import com.fasterxml.jackson.core.JsonParser.Feature;
|
import com.fasterxml.jackson.core.JsonParser.Feature;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
@ -87,12 +88,13 @@ public class JsonInputFormat extends NestedInputFormat
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_EMPTY)
|
||||||
public Map<String, Boolean> getFeatureSpec()
|
public Map<String, Boolean> getFeatureSpec()
|
||||||
{
|
{
|
||||||
return featureSpec;
|
return featureSpec;
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonProperty
|
@JsonProperty // No @JsonInclude, since default is variable, so we can't assume false is default
|
||||||
public boolean isKeepNullColumns()
|
public boolean isKeepNullColumns()
|
||||||
{
|
{
|
||||||
return keepNullColumns;
|
return keepNullColumns;
|
||||||
|
|
|
@ -20,6 +20,7 @@
|
||||||
package org.apache.druid.data.input.impl;
|
package org.apache.druid.data.input.impl;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
import com.google.common.base.Preconditions;
|
import com.google.common.base.Preconditions;
|
||||||
|
@ -86,6 +87,7 @@ public class LocalInputSource extends AbstractInputSource implements SplittableI
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public File getBaseDir()
|
public File getBaseDir()
|
||||||
{
|
{
|
||||||
return baseDir;
|
return baseDir;
|
||||||
|
@ -93,12 +95,14 @@ public class LocalInputSource extends AbstractInputSource implements SplittableI
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public String getFilter()
|
public String getFilter()
|
||||||
{
|
{
|
||||||
return filter;
|
return filter;
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_EMPTY)
|
||||||
public List<File> getFiles()
|
public List<File> getFiles()
|
||||||
{
|
{
|
||||||
return files;
|
return files;
|
||||||
|
|
|
@ -19,6 +19,7 @@
|
||||||
|
|
||||||
package org.apache.druid.data.input.impl;
|
package org.apache.druid.data.input.impl;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import org.apache.druid.data.input.InputFormat;
|
import org.apache.druid.data.input.InputFormat;
|
||||||
import org.apache.druid.java.util.common.parsers.JSONPathSpec;
|
import org.apache.druid.java.util.common.parsers.JSONPathSpec;
|
||||||
|
@ -42,6 +43,7 @@ public abstract class NestedInputFormat implements InputFormat
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
@JsonProperty("flattenSpec")
|
@JsonProperty("flattenSpec")
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public JSONPathSpec getFlattenSpec()
|
public JSONPathSpec getFlattenSpec()
|
||||||
{
|
{
|
||||||
return flattenSpec;
|
return flattenSpec;
|
||||||
|
|
|
@ -21,6 +21,7 @@ package org.apache.druid.data.input.impl;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import com.google.common.base.Supplier;
|
import com.google.common.base.Supplier;
|
||||||
import com.google.common.base.Suppliers;
|
import com.google.common.base.Suppliers;
|
||||||
|
@ -63,6 +64,7 @@ public class RegexInputFormat implements InputFormat
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public String getListDelimiter()
|
public String getListDelimiter()
|
||||||
{
|
{
|
||||||
return listDelimiter;
|
return listDelimiter;
|
||||||
|
@ -70,6 +72,7 @@ public class RegexInputFormat implements InputFormat
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public List<String> getColumns()
|
public List<String> getColumns()
|
||||||
{
|
{
|
||||||
return columns;
|
return columns;
|
||||||
|
|
|
@ -21,6 +21,7 @@ package org.apache.druid.data.input.avro;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JacksonInject;
|
import com.fasterxml.jackson.annotation.JacksonInject;
|
||||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import org.apache.avro.Schema;
|
import org.apache.avro.Schema;
|
||||||
|
@ -79,19 +80,23 @@ public class AvroOCFInputFormat extends NestedInputFormat
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Nullable
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public Map<String, Object> getSchema()
|
public Map<String, Object> getSchema()
|
||||||
{
|
{
|
||||||
return schema;
|
return schema;
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
public Boolean getBinaryAsString()
|
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
|
||||||
|
public boolean getBinaryAsString()
|
||||||
{
|
{
|
||||||
return binaryAsString;
|
return binaryAsString;
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
|
||||||
public Boolean isExtractUnionsByType()
|
public Boolean isExtractUnionsByType()
|
||||||
{
|
{
|
||||||
return extractUnionsByType;
|
return extractUnionsByType;
|
||||||
|
|
|
@ -218,6 +218,11 @@
|
||||||
<artifactId>junit</artifactId>
|
<artifactId>junit</artifactId>
|
||||||
<scope>test</scope>
|
<scope>test</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>nl.jqno.equalsverifier</groupId>
|
||||||
|
<artifactId>equalsverifier</artifactId>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.druid</groupId>
|
<groupId>org.apache.druid</groupId>
|
||||||
<artifactId>druid-core</artifactId>
|
<artifactId>druid-core</artifactId>
|
||||||
|
@ -225,6 +230,13 @@
|
||||||
<type>test-jar</type>
|
<type>test-jar</type>
|
||||||
<scope>test</scope>
|
<scope>test</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.druid</groupId>
|
||||||
|
<artifactId>druid-processing</artifactId>
|
||||||
|
<version>${project.parent.version}</version>
|
||||||
|
<type>test-jar</type>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
<profiles>
|
<profiles>
|
||||||
<profile>
|
<profile>
|
||||||
|
|
|
@ -21,6 +21,7 @@ package org.apache.druid.data.input.orc;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JacksonInject;
|
import com.fasterxml.jackson.annotation.JacksonInject;
|
||||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import org.apache.druid.data.input.InputEntity;
|
import org.apache.druid.data.input.InputEntity;
|
||||||
import org.apache.druid.data.input.InputEntityReader;
|
import org.apache.druid.data.input.InputEntityReader;
|
||||||
|
@ -49,7 +50,7 @@ public class OrcInputFormat extends NestedInputFormat
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
super(flattenSpec);
|
super(flattenSpec);
|
||||||
this.binaryAsString = binaryAsString == null ? false : binaryAsString;
|
this.binaryAsString = binaryAsString != null && binaryAsString;
|
||||||
this.conf = conf;
|
this.conf = conf;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -79,6 +80,13 @@ public class OrcInputFormat extends NestedInputFormat
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
|
||||||
|
public boolean getBinaryAsString()
|
||||||
|
{
|
||||||
|
return binaryAsString;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory)
|
public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory)
|
||||||
{
|
{
|
||||||
|
|
|
@ -0,0 +1,79 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.druid.data.input.orc;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.InjectableValues;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import nl.jqno.equalsverifier.EqualsVerifier;
|
||||||
|
import org.apache.druid.data.input.InputFormat;
|
||||||
|
import org.apache.druid.java.util.common.parsers.JSONPathSpec;
|
||||||
|
import org.apache.druid.segment.TestHelper;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import java.util.Collections;
|
||||||
|
|
||||||
|
public class OrcInputFormatTest
|
||||||
|
{
|
||||||
|
private ObjectMapper mapper;
|
||||||
|
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setUp()
|
||||||
|
{
|
||||||
|
mapper =
|
||||||
|
TestHelper.makeJsonMapper()
|
||||||
|
.registerModules(new OrcExtensionsModule().getJacksonModules())
|
||||||
|
.setInjectableValues(new InjectableValues.Std().addValue(Configuration.class, null));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSerdeDefault() throws Exception
|
||||||
|
{
|
||||||
|
final OrcInputFormat config = new OrcInputFormat(null, null, null);
|
||||||
|
|
||||||
|
Assert.assertEquals(
|
||||||
|
config,
|
||||||
|
mapper.readValue(mapper.writeValueAsString(config), InputFormat.class)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSerdeNonDefault() throws Exception
|
||||||
|
{
|
||||||
|
final OrcInputFormat config = new OrcInputFormat(new JSONPathSpec(true, Collections.emptyList()), true, null);
|
||||||
|
|
||||||
|
Assert.assertEquals(
|
||||||
|
config,
|
||||||
|
mapper.readValue(mapper.writeValueAsString(config), InputFormat.class)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testEquals()
|
||||||
|
{
|
||||||
|
EqualsVerifier.forClass(OrcInputFormat.class)
|
||||||
|
.withPrefabValues(Configuration.class, new Configuration(), new Configuration())
|
||||||
|
.usingGetClass()
|
||||||
|
.verify();
|
||||||
|
}
|
||||||
|
}
|
|
@ -21,6 +21,7 @@ package org.apache.druid.data.input.parquet;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JacksonInject;
|
import com.fasterxml.jackson.annotation.JacksonInject;
|
||||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import org.apache.druid.data.input.InputEntity;
|
import org.apache.druid.data.input.InputEntity;
|
||||||
import org.apache.druid.data.input.InputEntityReader;
|
import org.apache.druid.data.input.InputEntityReader;
|
||||||
|
@ -74,6 +75,7 @@ public class ParquetInputFormat extends NestedInputFormat
|
||||||
}
|
}
|
||||||
|
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
|
||||||
public boolean getBinaryAsString()
|
public boolean getBinaryAsString()
|
||||||
{
|
{
|
||||||
return binaryAsString;
|
return binaryAsString;
|
||||||
|
|
|
@ -168,6 +168,11 @@
|
||||||
<artifactId>hamcrest-core</artifactId>
|
<artifactId>hamcrest-core</artifactId>
|
||||||
<scope>test</scope>
|
<scope>test</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>nl.jqno.equalsverifier</groupId>
|
||||||
|
<artifactId>equalsverifier</artifactId>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
</project>
|
</project>
|
||||||
|
|
|
@ -30,6 +30,7 @@ import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
|
||||||
import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder;
|
import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder;
|
||||||
import com.fasterxml.jackson.annotation.JacksonInject;
|
import com.fasterxml.jackson.annotation.JacksonInject;
|
||||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
import com.google.common.base.Preconditions;
|
import com.google.common.base.Preconditions;
|
||||||
|
@ -77,7 +78,6 @@ public class S3InputSource extends CloudObjectInputSource
|
||||||
private final AWSProxyConfig awsProxyConfig;
|
private final AWSProxyConfig awsProxyConfig;
|
||||||
private final AWSClientConfig awsClientConfig;
|
private final AWSClientConfig awsClientConfig;
|
||||||
private final AWSEndpointConfig awsEndpointConfig;
|
private final AWSEndpointConfig awsEndpointConfig;
|
||||||
private final AWSCredentialsProvider awsCredentialsProvider;
|
|
||||||
private int maxRetries;
|
private int maxRetries;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -104,6 +104,7 @@ public class S3InputSource extends CloudObjectInputSource
|
||||||
@JacksonInject ServerSideEncryptingAmazonS3 s3Client,
|
@JacksonInject ServerSideEncryptingAmazonS3 s3Client,
|
||||||
@JacksonInject ServerSideEncryptingAmazonS3.Builder s3ClientBuilder,
|
@JacksonInject ServerSideEncryptingAmazonS3.Builder s3ClientBuilder,
|
||||||
@JacksonInject S3InputDataConfig inputDataConfig,
|
@JacksonInject S3InputDataConfig inputDataConfig,
|
||||||
|
@JacksonInject AWSCredentialsProvider awsCredentialsProvider,
|
||||||
@JsonProperty("uris") @Nullable List<URI> uris,
|
@JsonProperty("uris") @Nullable List<URI> uris,
|
||||||
@JsonProperty("prefixes") @Nullable List<URI> prefixes,
|
@JsonProperty("prefixes") @Nullable List<URI> prefixes,
|
||||||
@JsonProperty("objects") @Nullable List<CloudObjectLocation> objects,
|
@JsonProperty("objects") @Nullable List<CloudObjectLocation> objects,
|
||||||
|
@ -111,8 +112,7 @@ public class S3InputSource extends CloudObjectInputSource
|
||||||
@JsonProperty("properties") @Nullable S3InputSourceConfig s3InputSourceConfig,
|
@JsonProperty("properties") @Nullable S3InputSourceConfig s3InputSourceConfig,
|
||||||
@JsonProperty("proxyConfig") @Nullable AWSProxyConfig awsProxyConfig,
|
@JsonProperty("proxyConfig") @Nullable AWSProxyConfig awsProxyConfig,
|
||||||
@JsonProperty("endpointConfig") @Nullable AWSEndpointConfig awsEndpointConfig,
|
@JsonProperty("endpointConfig") @Nullable AWSEndpointConfig awsEndpointConfig,
|
||||||
@JsonProperty("clientConfig") @Nullable AWSClientConfig awsClientConfig,
|
@JsonProperty("clientConfig") @Nullable AWSClientConfig awsClientConfig
|
||||||
@JacksonInject AWSCredentialsProvider awsCredentialsProvider
|
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
super(S3StorageDruidModule.SCHEME, uris, prefixes, objects, filter);
|
super(S3StorageDruidModule.SCHEME, uris, prefixes, objects, filter);
|
||||||
|
@ -174,11 +174,10 @@ public class S3InputSource extends CloudObjectInputSource
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
this.maxRetries = RetryUtils.DEFAULT_MAX_TRIES;
|
this.maxRetries = RetryUtils.DEFAULT_MAX_TRIES;
|
||||||
this.awsCredentialsProvider = awsCredentialsProvider;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
public S3InputSource(
|
S3InputSource(
|
||||||
ServerSideEncryptingAmazonS3 s3Client,
|
ServerSideEncryptingAmazonS3 s3Client,
|
||||||
ServerSideEncryptingAmazonS3.Builder s3ClientBuilder,
|
ServerSideEncryptingAmazonS3.Builder s3ClientBuilder,
|
||||||
S3InputDataConfig inputDataConfig,
|
S3InputDataConfig inputDataConfig,
|
||||||
|
@ -192,18 +191,19 @@ public class S3InputSource extends CloudObjectInputSource
|
||||||
AWSClientConfig awsClientConfig
|
AWSClientConfig awsClientConfig
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
this(s3Client,
|
this(
|
||||||
s3ClientBuilder,
|
s3Client,
|
||||||
inputDataConfig,
|
s3ClientBuilder,
|
||||||
uris,
|
inputDataConfig,
|
||||||
prefixes,
|
null,
|
||||||
objects,
|
uris,
|
||||||
filter,
|
prefixes,
|
||||||
s3InputSourceConfig,
|
objects,
|
||||||
awsProxyConfig,
|
filter,
|
||||||
awsEndpointConfig,
|
s3InputSourceConfig,
|
||||||
awsClientConfig,
|
awsProxyConfig,
|
||||||
null
|
awsEndpointConfig,
|
||||||
|
awsClientConfig
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -227,6 +227,7 @@ public class S3InputSource extends CloudObjectInputSource
|
||||||
s3Client,
|
s3Client,
|
||||||
s3ClientBuilder,
|
s3ClientBuilder,
|
||||||
inputDataConfig,
|
inputDataConfig,
|
||||||
|
null,
|
||||||
uris,
|
uris,
|
||||||
prefixes,
|
prefixes,
|
||||||
objects,
|
objects,
|
||||||
|
@ -234,8 +235,7 @@ public class S3InputSource extends CloudObjectInputSource
|
||||||
s3InputSourceConfig,
|
s3InputSourceConfig,
|
||||||
awsProxyConfig,
|
awsProxyConfig,
|
||||||
awsEndpointConfig,
|
awsEndpointConfig,
|
||||||
awsClientConfig,
|
awsClientConfig
|
||||||
null
|
|
||||||
);
|
);
|
||||||
this.maxRetries = maxRetries;
|
this.maxRetries = maxRetries;
|
||||||
}
|
}
|
||||||
|
@ -278,6 +278,7 @@ public class S3InputSource extends CloudObjectInputSource
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
@JsonProperty("properties")
|
@JsonProperty("properties")
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public S3InputSourceConfig getS3InputSourceConfig()
|
public S3InputSourceConfig getS3InputSourceConfig()
|
||||||
{
|
{
|
||||||
return s3InputSourceConfig;
|
return s3InputSourceConfig;
|
||||||
|
@ -285,6 +286,7 @@ public class S3InputSource extends CloudObjectInputSource
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
@JsonProperty("proxyConfig")
|
@JsonProperty("proxyConfig")
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public AWSProxyConfig getAwsProxyConfig()
|
public AWSProxyConfig getAwsProxyConfig()
|
||||||
{
|
{
|
||||||
return awsProxyConfig;
|
return awsProxyConfig;
|
||||||
|
@ -292,6 +294,7 @@ public class S3InputSource extends CloudObjectInputSource
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
@JsonProperty("clientConfig")
|
@JsonProperty("clientConfig")
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public AWSClientConfig getAwsClientConfig()
|
public AWSClientConfig getAwsClientConfig()
|
||||||
{
|
{
|
||||||
return awsClientConfig;
|
return awsClientConfig;
|
||||||
|
@ -299,6 +302,7 @@ public class S3InputSource extends CloudObjectInputSource
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
@JsonProperty("endpointConfig")
|
@JsonProperty("endpointConfig")
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public AWSEndpointConfig getAwsEndpointConfig()
|
public AWSEndpointConfig getAwsEndpointConfig()
|
||||||
{
|
{
|
||||||
return awsEndpointConfig;
|
return awsEndpointConfig;
|
||||||
|
@ -334,13 +338,13 @@ public class S3InputSource extends CloudObjectInputSource
|
||||||
inputDataConfig,
|
inputDataConfig,
|
||||||
null,
|
null,
|
||||||
null,
|
null,
|
||||||
|
null,
|
||||||
split.get(),
|
split.get(),
|
||||||
getFilter(),
|
getFilter(),
|
||||||
getS3InputSourceConfig(),
|
getS3InputSourceConfig(),
|
||||||
getAwsProxyConfig(),
|
getAwsProxyConfig(),
|
||||||
getAwsEndpointConfig(),
|
getAwsEndpointConfig(),
|
||||||
getAwsClientConfig(),
|
getAwsClientConfig()
|
||||||
awsCredentialsProvider
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -21,6 +21,7 @@ package org.apache.druid.data.input.s3;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import com.google.common.base.Preconditions;
|
import com.google.common.base.Preconditions;
|
||||||
import org.apache.druid.metadata.PasswordProvider;
|
import org.apache.druid.metadata.PasswordProvider;
|
||||||
|
@ -34,14 +35,10 @@ import java.util.Objects;
|
||||||
*/
|
*/
|
||||||
public class S3InputSourceConfig
|
public class S3InputSourceConfig
|
||||||
{
|
{
|
||||||
@JsonProperty
|
private final String assumeRoleArn;
|
||||||
private String assumeRoleArn;
|
private final String assumeRoleExternalId;
|
||||||
@JsonProperty
|
private final PasswordProvider accessKeyId;
|
||||||
private String assumeRoleExternalId;
|
private final PasswordProvider secretAccessKey;
|
||||||
@JsonProperty
|
|
||||||
private PasswordProvider accessKeyId;
|
|
||||||
@JsonProperty
|
|
||||||
private PasswordProvider secretAccessKey;
|
|
||||||
|
|
||||||
@JsonCreator
|
@JsonCreator
|
||||||
public S3InputSourceConfig(
|
public S3InputSourceConfig(
|
||||||
|
@ -56,28 +53,39 @@ public class S3InputSourceConfig
|
||||||
if (accessKeyId != null || secretAccessKey != null) {
|
if (accessKeyId != null || secretAccessKey != null) {
|
||||||
this.accessKeyId = Preconditions.checkNotNull(accessKeyId, "accessKeyId cannot be null if secretAccessKey is given");
|
this.accessKeyId = Preconditions.checkNotNull(accessKeyId, "accessKeyId cannot be null if secretAccessKey is given");
|
||||||
this.secretAccessKey = Preconditions.checkNotNull(secretAccessKey, "secretAccessKey cannot be null if accessKeyId is given");
|
this.secretAccessKey = Preconditions.checkNotNull(secretAccessKey, "secretAccessKey cannot be null if accessKeyId is given");
|
||||||
|
} else {
|
||||||
|
this.accessKeyId = null;
|
||||||
|
this.secretAccessKey = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public String getAssumeRoleArn()
|
public String getAssumeRoleArn()
|
||||||
{
|
{
|
||||||
return assumeRoleArn;
|
return assumeRoleArn;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public String getAssumeRoleExternalId()
|
public String getAssumeRoleExternalId()
|
||||||
{
|
{
|
||||||
return assumeRoleExternalId;
|
return assumeRoleExternalId;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public PasswordProvider getAccessKeyId()
|
public PasswordProvider getAccessKeyId()
|
||||||
{
|
{
|
||||||
return accessKeyId;
|
return accessKeyId;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
|
@JsonProperty
|
||||||
|
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||||
public PasswordProvider getSecretAccessKey()
|
public PasswordProvider getSecretAccessKey()
|
||||||
{
|
{
|
||||||
return secretAccessKey;
|
return secretAccessKey;
|
||||||
|
|
|
@ -0,0 +1,70 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.druid.data.input.s3;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import nl.jqno.equalsverifier.EqualsVerifier;
|
||||||
|
import org.apache.druid.metadata.DefaultPasswordProvider;
|
||||||
|
import org.apache.druid.segment.TestHelper;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class S3InputSourceConfigTest
|
||||||
|
{
|
||||||
|
@Test
|
||||||
|
public void testSerdeAccessSecretKey() throws Exception
|
||||||
|
{
|
||||||
|
final ObjectMapper mapper = TestHelper.makeJsonMapper();
|
||||||
|
final S3InputSourceConfig config = new S3InputSourceConfig(
|
||||||
|
new DefaultPasswordProvider("the-access-key"),
|
||||||
|
new DefaultPasswordProvider("the-secret-key"),
|
||||||
|
null,
|
||||||
|
null
|
||||||
|
);
|
||||||
|
|
||||||
|
Assert.assertEquals(
|
||||||
|
config,
|
||||||
|
mapper.readValue(mapper.writeValueAsString(config), S3InputSourceConfig.class)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSerdeAssumeRole() throws Exception
|
||||||
|
{
|
||||||
|
final ObjectMapper mapper = TestHelper.makeJsonMapper();
|
||||||
|
final S3InputSourceConfig config = new S3InputSourceConfig(
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
"the-role-arn",
|
||||||
|
"the-role-external-id"
|
||||||
|
);
|
||||||
|
|
||||||
|
Assert.assertEquals(
|
||||||
|
config,
|
||||||
|
mapper.readValue(mapper.writeValueAsString(config), S3InputSourceConfig.class)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testEquals()
|
||||||
|
{
|
||||||
|
EqualsVerifier.forClass(S3InputSourceConfig.class).usingGetClass().verify();
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue