mirror of https://github.com/apache/druid.git
LocalInputSource: Serialize File paths without forcing resolution. (#13534)
* LocalInputSource: Serialize File paths without forcing resolution. Fixes #13359. * Add one more javadoc.
This commit is contained in:
parent
09d8b16447
commit
ee890965f4
|
@ -87,13 +87,28 @@ public class LocalInputSource extends AbstractInputSource implements SplittableI
|
|||
}
|
||||
|
||||
@Nullable
|
||||
@JsonProperty
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
public File getBaseDir()
|
||||
{
|
||||
return baseDir;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the base directory for serialization. This is better than returning {@link File} directly, because
|
||||
* Jackson serializes {@link File} using {@link File#getAbsolutePath()}, and we'd prefer to not force relative
|
||||
* path resolution as part of serialization.
|
||||
*/
|
||||
@Nullable
|
||||
@JsonProperty("baseDir")
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
private String getBaseDirForSerialization()
|
||||
{
|
||||
if (baseDir == null) {
|
||||
return null;
|
||||
} else {
|
||||
return baseDir.getPath();
|
||||
}
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@JsonProperty
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
|
@ -102,13 +117,23 @@ public class LocalInputSource extends AbstractInputSource implements SplittableI
|
|||
return filter;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
@JsonInclude(JsonInclude.Include.NON_EMPTY)
|
||||
public List<File> getFiles()
|
||||
{
|
||||
return files;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the list of file paths for serialization. This is better than returning {@link File} directly, because
|
||||
* Jackson serializes {@link File} using {@link File#getAbsolutePath()}, and we'd prefer to not force relative
|
||||
* path resolution as part of serialization.
|
||||
*/
|
||||
@JsonProperty("files")
|
||||
@JsonInclude(JsonInclude.Include.NON_EMPTY)
|
||||
private List<String> getFilesForSerialization()
|
||||
{
|
||||
return getFiles().stream().map(File::getPath).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Stream<InputSplit<List<File>>> createSplits(InputFormat inputFormat, @Nullable SplitHintSpec splitHintSpec)
|
||||
{
|
||||
|
@ -233,9 +258,9 @@ public class LocalInputSource extends AbstractInputSource implements SplittableI
|
|||
public String toString()
|
||||
{
|
||||
return "LocalInputSource{" +
|
||||
"baseDir=\"" + baseDir +
|
||||
"\", filter=" + filter +
|
||||
", files=" + files +
|
||||
"}";
|
||||
"baseDir=\"" + baseDir +
|
||||
"\", filter=" + filter +
|
||||
", files=" + files +
|
||||
"}";
|
||||
}
|
||||
}
|
||||
|
|
|
@ -20,6 +20,7 @@
|
|||
package org.apache.druid.data.input.impl;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.Lists;
|
||||
import nl.jqno.equalsverifier.EqualsVerifier;
|
||||
import org.apache.druid.data.input.InputSource;
|
||||
|
@ -51,7 +52,7 @@ public class LocalInputSourceTest
|
|||
public TemporaryFolder temporaryFolder = new TemporaryFolder();
|
||||
|
||||
@Test
|
||||
public void testSerde() throws IOException
|
||||
public void testSerdeAbsoluteBaseDir() throws IOException
|
||||
{
|
||||
final ObjectMapper mapper = new ObjectMapper();
|
||||
final LocalInputSource source = new LocalInputSource(new File("myFile").getAbsoluteFile(), "myFilter");
|
||||
|
@ -60,6 +61,33 @@ public class LocalInputSourceTest
|
|||
Assert.assertEquals(source, fromJson);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSerdeRelativeBaseDir() throws IOException
|
||||
{
|
||||
final ObjectMapper mapper = new ObjectMapper();
|
||||
final LocalInputSource source = new LocalInputSource(new File("myFile"), "myFilter");
|
||||
final byte[] json = mapper.writeValueAsBytes(source);
|
||||
final LocalInputSource fromJson = (LocalInputSource) mapper.readValue(json, InputSource.class);
|
||||
Assert.assertEquals(source, fromJson);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSerdeMixedAbsoluteAndRelativeFiles() throws IOException
|
||||
{
|
||||
final ObjectMapper mapper = new ObjectMapper();
|
||||
final LocalInputSource source = new LocalInputSource(
|
||||
null,
|
||||
null,
|
||||
ImmutableList.of(
|
||||
new File("myFile1"),
|
||||
new File("myFile2").getAbsoluteFile()
|
||||
)
|
||||
);
|
||||
final byte[] json = mapper.writeValueAsBytes(source);
|
||||
final LocalInputSource fromJson = (LocalInputSource) mapper.readValue(json, InputSource.class);
|
||||
Assert.assertEquals(source, fromJson);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEquals()
|
||||
{
|
||||
|
|
|
@ -134,10 +134,6 @@ You must either provide the `baseDir` or the list of `files`. You can provide bo
|
|||
the files are assumed relative to the `baseDir`. If you provide a `filter`, you must provide the
|
||||
`baseDir`.
|
||||
|
||||
Note that, due to [Issue #13359](https://github.com/apache/druid/issues/13359), the functionality
|
||||
described above is broken. Until that issue is resolved, you must provide one or more absolute
|
||||
file paths in the `files` property and the other two properties are unavailable.
|
||||
|
||||
#### Table Function Format
|
||||
|
||||
Each of the table functions above requires that you specify a format.
|
||||
|
|
Loading…
Reference in New Issue