nodes will not try and download segments during startup

This commit is contained in:
Fangjin Yang 2013-04-01 16:55:03 -07:00 committed by Gian Merlino
parent 2fcf7e018b
commit 917cb20494
7 changed files with 44 additions and 2 deletions

View File

@ -104,6 +104,11 @@ public class ServerManager implements QuerySegmentWalker
} }
} }
public boolean isSegmentLoaded(final DataSegment segment) throws SegmentLoadingException
{
return segmentLoader.isSegmentLoaded(segment);
}
public void loadSegment(final DataSegment segment) throws SegmentLoadingException public void loadSegment(final DataSegment segment) throws SegmentLoadingException
{ {
final Segment adapter; final Segment adapter;

View File

@ -215,9 +215,12 @@ public class ZkCoordinator implements DataSegmentChangeHandler
for (File file : baseDir.listFiles()) { for (File file : baseDir.listFiles()) {
log.info("Loading segment cache file [%s]", file); log.info("Loading segment cache file [%s]", file);
try { try {
addSegment(jsonMapper.readValue(file, DataSegment.class)); DataSegment segment = jsonMapper.readValue(file, DataSegment.class);
if (serverManager.isSegmentLoaded(segment)) {
addSegment(segment);
}
} }
catch (IOException e) { catch (Exception e) {
log.error(e, "Exception occurred reading file [%s]", file); log.error(e, "Exception occurred reading file [%s]", file);
emitter.emit( emitter.emit(
new AlertEvent.Builder().build( new AlertEvent.Builder().build(

View File

@ -43,6 +43,12 @@ public class DelegatingSegmentLoader implements SegmentLoader
this.loaderTypes = loaderTypes; this.loaderTypes = loaderTypes;
} }
@Override
public boolean isSegmentLoaded(DataSegment segment) throws SegmentLoadingException
{
return getLoader(segment.getLoadSpec()).isSegmentLoaded(segment);
}
@Override @Override
public Segment getSegment(DataSegment segment) throws SegmentLoadingException public Segment getSegment(DataSegment segment) throws SegmentLoadingException
{ {

View File

@ -26,6 +26,7 @@ import com.metamx.druid.index.Segment;
*/ */
public interface SegmentLoader public interface SegmentLoader
{ {
public boolean isSegmentLoaded(DataSegment segment) throws SegmentLoadingException;
public Segment getSegment(DataSegment loadSpec) throws SegmentLoadingException; public Segment getSegment(DataSegment loadSpec) throws SegmentLoadingException;
public void cleanup(DataSegment loadSpec) throws SegmentLoadingException; public void cleanup(DataSegment loadSpec) throws SegmentLoadingException;
} }

View File

@ -55,6 +55,21 @@ public class SingleSegmentLoader implements SegmentLoader
this.config = config; this.config = config;
} }
@Override
public boolean isSegmentLoaded(final DataSegment segment)
{
File localStorageDir = new File(config.getCacheDirectory(), DataSegmentPusherUtil.getStorageDir(segment));
if (localStorageDir.exists()) {
return true;
}
final File legacyStorageDir = new File(
config.getCacheDirectory(),
DataSegmentPusherUtil.getLegacyStorageDir(segment)
);
return legacyStorageDir.exists();
}
@Override @Override
public Segment getSegment(DataSegment segment) throws SegmentLoadingException public Segment getSegment(DataSegment segment) throws SegmentLoadingException
{ {

View File

@ -84,6 +84,12 @@ public class ServerManagerTest
serverManager = new ServerManager( serverManager = new ServerManager(
new SegmentLoader() new SegmentLoader()
{ {
@Override
public boolean isSegmentLoaded(DataSegment segment) throws SegmentLoadingException
{
return false;
}
@Override @Override
public Segment getSegment(final DataSegment segment) public Segment getSegment(final DataSegment segment)
{ {

View File

@ -29,6 +29,12 @@ import org.joda.time.Interval;
*/ */
public class NoopSegmentLoader implements SegmentLoader public class NoopSegmentLoader implements SegmentLoader
{ {
@Override
public boolean isSegmentLoaded(DataSegment segment) throws SegmentLoadingException
{
return false;
}
@Override @Override
public Segment getSegment(final DataSegment segment) throws SegmentLoadingException public Segment getSegment(final DataSegment segment) throws SegmentLoadingException
{ {