Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

1) Adjust the SingleSegmentLoader to be able to use cache locations from previous versions as well without re-downloading

This commit is contained in:
Eric Tschetter 2013-03-07 15:55:23 -06:00
parent bb1b3cd2f9
commit 82aa6ed7dd
3 changed files with 39 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
server/src/main/java/com/metamx/druid/loading/DataSegmentPusherUtil.java
View File

@ -20,14 +20,32 @@
package com.metamx.druid.loading;
import com.google.common.base.Joiner;
import com.metamx.common.MapUtils;
import com.metamx.druid.client.DataSegment;
import java.util.Map;
/**
*/
public class DataSegmentPusherUtil
{
private static final Joiner JOINER = Joiner.on("/").skipNulls();
public static String getLegacyStorageDir(DataSegment segment)
{
final Map<String,Object> loadSpec = segment.getLoadSpec();
String specType = MapUtils.getString(loadSpec, "type");
if (specType.startsWith("s3")) {
String s3Bucket = MapUtils.getString(loadSpec, "bucket");
String s3Path = MapUtils.getString(loadSpec, "key");
return String.format("%s/%s", s3Bucket, s3Path.substring(0, s3Path.lastIndexOf("/")));
}
return null;
}
public static String getStorageDir(DataSegment segment)
{
return JOINER.join(

2
server/src/main/java/com/metamx/druid/loading/MMappedQueryableIndexFactory.java
View File

@ -59,7 +59,7 @@ public class MMappedQueryableIndexFactory implements QueryableIndexFactory
return IndexIO.loadIndex(parentDir);
}
catch (IOException e) {
log.warn(e, "Got exception, deleting parentDir[%s]", parentDir);
log.warn(e, "Got exception!!!! Going to delete parentDir[%s]", parentDir);
try {
FileUtils.deleteDirectory(parentDir);
}

20
server/src/main/java/com/metamx/druid/loading/SingleSegmentLoader.java
View File

@ -67,6 +67,26 @@ public class SingleSegmentLoader implements SegmentLoader
public File getSegmentFiles(DataSegment segment) throws SegmentLoadingException
{
File localStorageDir = new File(config.getCacheDirectory(), DataSegmentPusherUtil.getStorageDir(segment));
final String legacyDir = DataSegmentPusherUtil.getLegacyStorageDir(segment);
if (legacyDir != null) {
File legacyStorageDir = new File(config.getCacheDirectory(), legacyDir);
if (legacyStorageDir.exists()) {
log.info("Found legacyStorageDir[%s], moving to new storage location[%s]", legacyStorageDir, localStorageDir);
if (localStorageDir.exists()) {
try {
FileUtils.deleteDirectory(localStorageDir);
}
catch (IOException e) {
throw new SegmentLoadingException(e, "Error deleting localDir[%s]", localStorageDir);
}
}
legacyStorageDir.renameTo(localStorageDir);
}
}
if (localStorageDir.exists()) {
long localLastModified = localStorageDir.lastModified();
long remoteLastModified = dataSegmentPuller.getLastModified(segment);