LocalInputSource: Serialize File paths without forcing resolution. (#13534)

* LocalInputSource: Serialize File paths without forcing resolution.

Fixes #13359.

* Add one more javadoc.
This commit is contained in:
Gian Merlino 2022-12-18 22:17:36 -08:00 committed by GitHub
parent 09d8b16447
commit ee890965f4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 62 additions and 13 deletions

View File

@ -87,13 +87,28 @@ public class LocalInputSource extends AbstractInputSource implements SplittableI
}
@Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
public File getBaseDir()
{
return baseDir;
}
/**
* Returns the base directory for serialization. This is better than returning {@link File} directly, because
* Jackson serializes {@link File} using {@link File#getAbsolutePath()}, and we'd prefer to not force relative
* path resolution as part of serialization.
*/
@Nullable
@JsonProperty("baseDir")
@JsonInclude(JsonInclude.Include.NON_NULL)
private String getBaseDirForSerialization()
{
if (baseDir == null) {
return null;
} else {
return baseDir.getPath();
}
}
@Nullable
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_NULL)
@ -102,13 +117,23 @@ public class LocalInputSource extends AbstractInputSource implements SplittableI
return filter;
}
@JsonProperty
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public List<File> getFiles()
{
return files;
}
/**
* Returns the list of file paths for serialization. This is better than returning {@link File} directly, because
* Jackson serializes {@link File} using {@link File#getAbsolutePath()}, and we'd prefer to not force relative
* path resolution as part of serialization.
*/
@JsonProperty("files")
@JsonInclude(JsonInclude.Include.NON_EMPTY)
private List<String> getFilesForSerialization()
{
return getFiles().stream().map(File::getPath).collect(Collectors.toList());
}
@Override
public Stream<InputSplit<List<File>>> createSplits(InputFormat inputFormat, @Nullable SplitHintSpec splitHintSpec)
{
@ -233,9 +258,9 @@ public class LocalInputSource extends AbstractInputSource implements SplittableI
public String toString()
{
return "LocalInputSource{" +
"baseDir=\"" + baseDir +
"\", filter=" + filter +
", files=" + files +
"}";
"baseDir=\"" + baseDir +
"\", filter=" + filter +
", files=" + files +
"}";
}
}

View File

@ -20,6 +20,7 @@
package org.apache.druid.data.input.impl;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.data.input.InputSource;
@ -51,7 +52,7 @@ public class LocalInputSourceTest
public TemporaryFolder temporaryFolder = new TemporaryFolder();
@Test
public void testSerde() throws IOException
public void testSerdeAbsoluteBaseDir() throws IOException
{
final ObjectMapper mapper = new ObjectMapper();
final LocalInputSource source = new LocalInputSource(new File("myFile").getAbsoluteFile(), "myFilter");
@ -60,6 +61,33 @@ public class LocalInputSourceTest
Assert.assertEquals(source, fromJson);
}
@Test
public void testSerdeRelativeBaseDir() throws IOException
{
final ObjectMapper mapper = new ObjectMapper();
final LocalInputSource source = new LocalInputSource(new File("myFile"), "myFilter");
final byte[] json = mapper.writeValueAsBytes(source);
final LocalInputSource fromJson = (LocalInputSource) mapper.readValue(json, InputSource.class);
Assert.assertEquals(source, fromJson);
}
@Test
public void testSerdeMixedAbsoluteAndRelativeFiles() throws IOException
{
final ObjectMapper mapper = new ObjectMapper();
final LocalInputSource source = new LocalInputSource(
null,
null,
ImmutableList.of(
new File("myFile1"),
new File("myFile2").getAbsoluteFile()
)
);
final byte[] json = mapper.writeValueAsBytes(source);
final LocalInputSource fromJson = (LocalInputSource) mapper.readValue(json, InputSource.class);
Assert.assertEquals(source, fromJson);
}
@Test
public void testEquals()
{

View File

@ -134,10 +134,6 @@ You must either provide the `baseDir` or the list of `files`. You can provide bo
the files are assumed relative to the `baseDir`. If you provide a `filter`, you must provide the
`baseDir`.
Note that, due to [Issue #13359](https://github.com/apache/druid/issues/13359), the functionality
described above is broken. Until that issue is resolved, you must provide one or more absolute
file paths in the `files` property and the other two properties are unavailable.
#### Table Function Format
Each of the table functions above requires that you specify a format.