Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update Delta Kernel to 3.2.1 (#17179) (#17198) 

Updated Delta Kernel from 3.2.0 to 3.2.1. This upstream version bump contains fixes to reading long columns, class loader and better retry mechanism when reading checkpoint files.
This commit is contained in:
Abhishek Radhakrishnan 2024-09-30 11:50:04 -07:00 committed by GitHub
parent c6f41dcd22
commit 8531a84d6c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 5 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
extensions-contrib/druid-deltalake-extensions/pom.xml
View File

@ -35,7 +35,7 @@
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<properties> <properties>
<delta-kernel.version>3.2.0</delta-kernel.version> <delta-kernel.version>3.2.1</delta-kernel.version>
</properties> </properties>
<dependencies> <dependencies>
@ -49,12 +49,6 @@
<artifactId>delta-kernel-defaults</artifactId> <artifactId>delta-kernel-defaults</artifactId>
<version>${delta-kernel.version}</version> <version>${delta-kernel.version}</version>
</dependency> </dependency>
<dependency>
<groupId>io.delta</groupId>
<artifactId>delta-storage</artifactId>
<version>${delta-kernel.version}</version>
</dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId> <artifactId>hadoop-client-api</artifactId>

19
extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputSource.java
View File

@ -42,7 +42,6 @@ import io.delta.kernel.types.StructField;
import io.delta.kernel.types.StructType; import io.delta.kernel.types.StructType;
import io.delta.kernel.utils.CloseableIterator; import io.delta.kernel.utils.CloseableIterator;
import io.delta.kernel.utils.FileStatus; import io.delta.kernel.utils.FileStatus;
import io.delta.storage.LogStore;
import org.apache.druid.data.input.ColumnsFilter; import org.apache.druid.data.input.ColumnsFilter;
import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.InputFormat;
import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.InputRowSchema;
@ -340,20 +339,10 @@ public class DeltaInputSource implements SplittableInputSource<DeltaSplit>
private Snapshot getSnapshotForTable(final Table table, final Engine engine) private Snapshot getSnapshotForTable(final Table table, final Engine engine)
{ {
// Setting the LogStore class loader before calling the Delta Kernel snapshot API is required as a workaround with if (snapshotVersion != null) {
// the 3.2.0 Delta Kernel because the Kernel library cannot instantiate the LogStore class otherwise. Please see return table.getSnapshotAsOfVersion(engine, snapshotVersion);
// https://github.com/delta-io/delta/issues/3299 for details. This workaround can be removed once the issue is fixed. } else {
final ClassLoader currCtxCl = Thread.currentThread().getContextClassLoader(); return table.getLatestSnapshot(engine);
try {
Thread.currentThread().setContextClassLoader(LogStore.class.getClassLoader());
if (snapshotVersion != null) {
return table.getSnapshotAsOfVersion(engine, snapshotVersion);
} else {
return table.getLatestSnapshot(engine);
}
}
finally {
Thread.currentThread().setContextClassLoader(currCtxCl);
} }
} }