fixes from review comments

fix sync of aggs,
fix NPE in sink.isFull,
RealtimeTuningConfig lower the bufferSize to 256m
This commit is contained in:
nishantmonu51 2014-08-21 09:24:04 +05:30
parent d64879ccca
commit 67f4bbae74
3 changed files with 165 additions and 124 deletions

View File

@ -130,7 +130,8 @@ public class IncrementalIndex implements Iterable<Row>, Closeable
int currAggSize = 0;
for (int i = 0; i < metrics.length; i++) {
final AggregatorFactory agg = metrics[i];
aggs[i] = agg.factorizeBuffered(
aggs[i] = new ThreadSafeAggregator(
agg.factorizeBuffered(
new ColumnSelectorFactory()
{
@Override
@ -270,6 +271,7 @@ public class IncrementalIndex implements Iterable<Row>, Closeable
};
}
}
)
);
aggPositionOffsets[i] = currAggSize;
currAggSize += agg.getMaxIntermediateSize();
@ -458,10 +460,8 @@ public class IncrementalIndex implements Iterable<Row>, Closeable
in.set(row);
int rowOffset = facts.get(key);
for (int i = 0; i < aggs.length; i++) {
synchronized (aggs[i]) {
aggs[i].aggregate(bufferHolder.get(), getMetricPosition(rowOffset, i));
}
}
in.set(null);
return numEntries.get();
}
@ -473,7 +473,7 @@ public class IncrementalIndex implements Iterable<Row>, Closeable
/**
*
* @return true if the underlying buffer for IncrementalIndex is full and cannot accomodate more rows.
* @return true if the underlying buffer for IncrementalIndex is full and cannot accommodate more rows.
*/
public boolean isFull()
{
@ -861,4 +861,45 @@ public class IncrementalIndex implements Iterable<Row>, Closeable
}
}
}
private static class ThreadSafeAggregator implements BufferAggregator
{
private final BufferAggregator delegate;
public ThreadSafeAggregator(BufferAggregator delegate)
{
this.delegate = delegate;
}
@Override
public synchronized void init(ByteBuffer buf, int position)
{
delegate.init(buf, position);
}
@Override
public synchronized void aggregate(ByteBuffer buf, int position)
{
delegate.aggregate(buf, position);
}
@Override
public synchronized Object get(ByteBuffer buf, int position)
{
return delegate.get(buf, position);
}
@Override
public synchronized float getFloat(ByteBuffer buf, int position)
{
return delegate.getFloat(buf, position);
}
@Override
public synchronized void close()
{
delegate.close();
}
}
}

View File

@ -36,7 +36,7 @@ import java.io.File;
*/
public class RealtimeTuningConfig implements TuningConfig
{
private static final int defaultBufferSize = 512 * 1024 * 1024;
private static final int defaultBufferSize = 256 * 1024 * 1024;
private static final Period defaultIntermediatePersistPeriod = new Period("PT10M");
private static final Period defaultWindowPeriod = new Period("PT10M");
private static final File defaultBasePersistDirectory = Files.createTempDir();

View File

@ -136,7 +136,7 @@ public class Sink implements Iterable<FireHydrant>
public boolean isFull()
{
synchronized (currHydrant){
return currHydrant.getIndex().isFull();
return currHydrant != null && currHydrant.getIndex().isFull();
}
}