Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Change last update timestamp granularity of GCS objects from seconds to milliseconds (#16083) 

The previously used GCS API client library returned last update time for objects directly in milliseconds. The new library returns it in OffsetDateTime format which was being converted to seconds and stored against the object. This fix converts the time back to ms before storing it.
This commit is contained in:
Vishesh Garg 2024-03-09 07:54:33 +05:30 committed by GitHub
parent 3aec90563e
commit b1c1937e94
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 47 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java
View File

@ -141,7 +141,7 @@ public class GoogleStorage
blob.getName(),
blob.getSize(),
blob.getUpdateTimeOffsetDateTime()
.toEpochSecond()
.toEpochSecond() * 1000
);
}
@ -250,7 +250,7 @@ public class GoogleStorage
blob.getName(),
blob.getSize(),
blob.getUpdateTimeOffsetDateTime()
.toEpochSecond()
.toEpochSecond() * 1000
))
.collect(Collectors.toList());

26
extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageObjectMetadata.java
View File

@ -26,19 +26,24 @@ public class GoogleStorageObjectMetadata
final String bucket;
final String name;
final Long size;
Long lastUpdateTime;
Long lastUpdateTimeMillis;
public GoogleStorageObjectMetadata(final String bucket, final String name, final Long size, final Long lastUpdateTime)
public GoogleStorageObjectMetadata(
final String bucket,
final String name,
final Long size,
final Long lastUpdateTimeMillis
)
{
this.bucket = bucket;
this.name = name;
this.size = size;
this.lastUpdateTime = lastUpdateTime;
this.lastUpdateTimeMillis = lastUpdateTimeMillis;
}
public void setLastUpdateTime(Long lastUpdateTime)
public void setLastUpdateTimeMillis(Long lastUpdateTimeMillis)
{
this.lastUpdateTime = lastUpdateTime;
this.lastUpdateTimeMillis = lastUpdateTimeMillis;
}
@ -57,9 +62,9 @@ public class GoogleStorageObjectMetadata
return size;
}
public Long getLastUpdateTime()
public Long getLastUpdateTimeMillis()
{
return lastUpdateTime;
return lastUpdateTimeMillis;
}
@Override
@ -74,13 +79,14 @@ public class GoogleStorageObjectMetadata
GoogleStorageObjectMetadata that = (GoogleStorageObjectMetadata) o;
return Objects.equals(bucket, that.bucket)
&& Objects.equals(name, that.name)
&& Objects.equals(size, that.size);
&& Objects.equals(size, that.size)
&& Objects.equals(lastUpdateTimeMillis, that.getLastUpdateTimeMillis());
}
@Override
public int hashCode()
{
return Objects.hash(bucket, name, size);
return Objects.hash(bucket, name, size, lastUpdateTimeMillis);
}
@Override
@ -90,7 +96,7 @@ public class GoogleStorageObjectMetadata
"bucket='" + bucket + '\'' +
", name='" + name + '\'' +
", size=" + size +
", lastUpdateTime=" + lastUpdateTime +
", lastUpdateTimeMillis=" + lastUpdateTimeMillis +
'}';
}
}

6
extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java
View File

@ -190,13 +190,13 @@ public class GoogleTaskLogs implements TaskLogs
}
@Override
public void killOlderThan(long timestamp) throws IOException
public void killOlderThan(long timestampMs) throws IOException
{
LOG.info(
"Deleting all task logs from gs location [bucket: '%s' prefix: '%s'] older than %s.",
config.getBucket(),
config.getPrefix(),
new Date(timestamp)
new Date(timestampMs)
);
try {
GoogleUtils.deleteObjectsInPath(
@ -204,7 +204,7 @@ public class GoogleTaskLogs implements TaskLogs
inputDataConfig,
config.getBucket(),
config.getPrefix(),
(object) -> object.getLastUpdateTime() < timestamp
(object) -> object.getLastUpdateTimeMillis() < timestampMs
);
}
catch (Exception e) {

2
extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinder.java
View File

@ -67,7 +67,7 @@ public class GoogleTimestampVersionedDataFinder extends GoogleDataSegmentPuller
if (pattern != null && !pattern.matcher(keyString).matches()) {
continue;
}
final long latestModified = objectMetadata.getLastUpdateTime();
final long latestModified = objectMetadata.getLastUpdateTimeMillis();
if (latestModified >= mostRecent) {
mostRecent = latestModified;
latest = objectLocation.toUri(GoogleStorageDruidModule.SCHEME_GS);

11
extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleStorageTest.java
View File

@ -143,7 +143,7 @@ public class GoogleStorageTest
}
@Test
public void testGetMetadata() throws IOException
public void testGetMetadataMatch() throws IOException
{
EasyMock.expect(mockStorage.get(
EasyMock.eq(BUCKET),
@ -159,7 +159,10 @@ public class GoogleStorageTest
EasyMock.replay(mockStorage, blob);
GoogleStorageObjectMetadata objectMetadata = googleStorage.getMetadata(BUCKET, PATH);
assertEquals(objectMetadata, new GoogleStorageObjectMetadata(BUCKET, PATH, SIZE, UPDATE_TIME.toEpochSecond()));
assertEquals(
objectMetadata,
new GoogleStorageObjectMetadata(BUCKET, PATH, SIZE, UPDATE_TIME.toEpochSecond() * 1000)
);
}
@ -262,13 +265,13 @@ public class GoogleStorageTest
bucket1,
path1,
size1,
updateTime1.toEpochSecond()
updateTime1.toEpochSecond() * 1000
);
GoogleStorageObjectMetadata objectMetadata2 = new GoogleStorageObjectMetadata(
bucket2,
path2,
size2,
updateTime2.toEpochSecond()
updateTime2.toEpochSecond() * 1000
);
GoogleStorageObjectPage objectPage = googleStorage.list(BUCKET, PATH, null, null);

16
extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTimestampVersionedDataFinderTest.java
View File

@ -37,13 +37,13 @@ public class GoogleTimestampVersionedDataFinderTest
// object for directory prefix/dir/0/
final GoogleStorageObjectMetadata storageObject1 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "//", 0);
storageObject1.setLastUpdateTime(System.currentTimeMillis());
storageObject1.setLastUpdateTimeMillis(System.currentTimeMillis());
final GoogleStorageObjectMetadata storageObject2 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/v1", 1);
storageObject2.setLastUpdateTime(System.currentTimeMillis());
storageObject2.setLastUpdateTimeMillis(System.currentTimeMillis());
final GoogleStorageObjectMetadata storageObject3 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/v2", 1);
storageObject3.setLastUpdateTime(System.currentTimeMillis() + 100);
storageObject3.setLastUpdateTimeMillis(System.currentTimeMillis() + 100);
final GoogleStorageObjectMetadata storageObject4 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/other", 4);
storageObject4.setLastUpdateTime(System.currentTimeMillis() + 100);
storageObject4.setLastUpdateTimeMillis(System.currentTimeMillis() + 100);
final GoogleStorage storage = ObjectStorageIteratorTest.makeMockClient(ImmutableList.of(storageObject1, storageObject2, storageObject3, storageObject4));
final GoogleTimestampVersionedDataFinder finder = new GoogleTimestampVersionedDataFinder(storage);
@ -61,13 +61,13 @@ public class GoogleTimestampVersionedDataFinderTest
// object for directory prefix/dir/0/
final GoogleStorageObjectMetadata storageObject1 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/", 0);
storageObject1.setLastUpdateTime(System.currentTimeMillis());
storageObject1.setLastUpdateTimeMillis(System.currentTimeMillis());
final GoogleStorageObjectMetadata storageObject2 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "v1", 1);
storageObject2.setLastUpdateTime(System.currentTimeMillis());
storageObject2.setLastUpdateTimeMillis(System.currentTimeMillis());
final GoogleStorageObjectMetadata storageObject3 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "v2", 1);
storageObject3.setLastUpdateTime(System.currentTimeMillis() + 100);
storageObject3.setLastUpdateTimeMillis(System.currentTimeMillis() + 100);
final GoogleStorageObjectMetadata storageObject4 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "other", 4);
storageObject4.setLastUpdateTime(System.currentTimeMillis() + 100);
storageObject4.setLastUpdateTimeMillis(System.currentTimeMillis() + 100);
final GoogleStorage storage = ObjectStorageIteratorTest.makeMockClient(ImmutableList.of(storageObject1, storageObject2, storageObject3, storageObject4));
final GoogleTimestampVersionedDataFinder finder = new GoogleTimestampVersionedDataFinder(storage);

4
indexing-service/src/main/java/org/apache/druid/indexing/overlord/duty/TaskLogAutoCleanerConfig.java
View File

@ -45,6 +45,10 @@ public class TaskLogAutoCleanerConfig
@JsonProperty
private final long durationToRetain;
/**
* Config for Task logs auto-cleaner.
* All time-related parameters should be in milliseconds.
*/
@JsonCreator
public TaskLogAutoCleanerConfig(
@JsonProperty("enabled") boolean enabled,

6
processing/src/main/java/org/apache/druid/tasklogs/TaskLogKiller.java
View File

@ -30,5 +30,11 @@ import java.io.IOException;
public interface TaskLogKiller
{
void killAll() throws IOException;
/**
* Removes logs older than the provided timestamp
* @param timestamp Timestamp in milliseconds
* @throws IOException
*/
void killOlderThan(long timestamp) throws IOException;
}