Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

QueryableIndexStorageAdapter: Lift column cache to Cursor sequence. (#4950) 

* QueryableIndexStorageAdapter: Lift column cache to Cursor sequence.

This is where it was before #4710, when its was moved to the individual
Cursors, leading to higher than expected memory usage. It could be
extreme for finer query granularities like "second".

* Comment.
This commit is contained in:
Gian Merlino 2017-10-12 14:44:33 -07:00 committed by Roman Leventov
parent d95915f8d2
commit 32f36beaae
2 changed files with 22 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
processing/src/main/java/io/druid/segment/QueryableIndexColumnSelectorFactory.java
View File

@ -19,7 +19,6 @@
package io.druid.segment;
import com.google.common.collect.Maps;
import io.druid.java.util.common.io.Closer;
import io.druid.query.dimension.DimensionSpec;
import io.druid.query.extraction.ExtractionFn;
@ -48,16 +47,19 @@ class QueryableIndexColumnSelectorFactory implements ColumnSelectorFactory
private final Closer closer;
protected final ReadableOffset offset;
private final Map<String, DictionaryEncodedColumn> dictionaryColumnCache = Maps.newHashMap();
private final Map<String, GenericColumn> genericColumnCache = Maps.newHashMap();
private final Map<String, Object> objectColumnCache = Maps.newHashMap();
private final Map<String, DictionaryEncodedColumn> dictionaryColumnCache;
private final Map<String, GenericColumn> genericColumnCache;
private final Map<String, Object> objectColumnCache;
QueryableIndexColumnSelectorFactory(
QueryableIndex index,
VirtualColumns virtualColumns,
boolean descending,
Closer closer,
ReadableOffset offset
ReadableOffset offset,
Map<String, DictionaryEncodedColumn> dictionaryColumnCache,
Map<String, GenericColumn> genericColumnCache,
Map<String, Object> objectColumnCache
)
{
this.index = index;
@ -65,6 +67,9 @@ class QueryableIndexColumnSelectorFactory implements ColumnSelectorFactory
this.descending = descending;
this.closer = closer;
this.offset = offset;
this.dictionaryColumnCache = dictionaryColumnCache;
this.genericColumnCache = genericColumnCache;
this.objectColumnCache = objectColumnCache;
}
@Override

13
processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java
View File

@ -39,6 +39,7 @@ import io.druid.segment.column.BitmapIndex;
import io.druid.segment.column.Column;
import io.druid.segment.column.ColumnCapabilities;
import io.druid.segment.column.ComplexColumn;
import io.druid.segment.column.DictionaryEncodedColumn;
import io.druid.segment.column.GenericColumn;
import io.druid.segment.data.Indexed;
import io.druid.segment.data.Offset;
@ -51,7 +52,9 @@ import org.joda.time.Interval;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
/**
@ -362,6 +365,11 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
{
final Offset baseOffset = offset.clone();
// Column caches shared amongst all cursors in this sequence.
final Map<String, DictionaryEncodedColumn> dictionaryColumnCache = new HashMap<>();
final Map<String, GenericColumn> genericColumnCache = new HashMap<>();
final Map<String, Object> objectColumnCache = new HashMap<>();
final GenericColumn timestamps = index.getColumn(Column.TIME_COLUMN_NAME).getGenericColumn();
final Closer closer = Closer.create();
@ -421,7 +429,10 @@ public class QueryableIndexStorageAdapter implements StorageAdapter
virtualColumns,
descending,
closer,
baseCursorOffset.getBaseReadableOffset()
baseCursorOffset.getBaseReadableOffset(),
dictionaryColumnCache,
genericColumnCache,
objectColumnCache
);
final DateTime myBucket = gran.toDateTime(inputInterval.getStartMillis());