Apache/druid
1
0
Fork 0
mirror of https://github.com/apache/druid.git synced 2025-02-09 11:34:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix binary serialization in caching (#4993)

Browse Source 
* Fix binary serialization in caching

The previous caching code just concatenated a list of objects into a
byte array -- this is actually not valid because jackson-databind uses
enumerated references to strings internally, and concatenating multiple
binary serialized objects can throw off the references.

This change uses a single JsonGenerator to serialize the object list
rather than concatenating byte arrays.

* remove unused imports
This commit is contained in:
Andy Sloane 2017-10-23 12:10:24 -07:00 committed by Gian Merlino
parent 772ca783cd
commit ee66db900e
2 changed files with 16 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
server/src/main/java/io/druid/client/CacheUtil.java
View File

@ -19,9 +19,9 @@
package io.druid.client; package io.druid.client;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Throwables; import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import io.druid.client.cache.Cache; import io.druid.client.cache.Cache;
import io.druid.client.cache.CacheConfig; import io.druid.client.cache.CacheConfig;
import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.StringUtils;
@ -31,9 +31,9 @@ import io.druid.query.QueryContexts;
import io.druid.query.SegmentDescriptor; import io.druid.query.SegmentDescriptor;
import org.joda.time.Interval; import org.joda.time.Interval;
import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.List;
public class CacheUtil public class CacheUtil
{ {
@ -60,22 +60,14 @@ public class CacheUtil
public static void populate(Cache cache, ObjectMapper mapper, Cache.NamedKey key, Iterable<Object> results) public static void populate(Cache cache, ObjectMapper mapper, Cache.NamedKey key, Iterable<Object> results)
{ {
try { try {
List<byte[]> bytes = Lists.newArrayList(); ByteArrayOutputStream bytes = new ByteArrayOutputStream();
int size = 0; try (JsonGenerator gen = mapper.getFactory().createGenerator(bytes)) {
for (Object result : results) { for (Object result : results) {
final byte[] array = mapper.writeValueAsBytes(result); gen.writeObject(result);
size += array.length; }
bytes.add(array);
} }
byte[] valueBytes = new byte[size]; cache.put(key, bytes.toByteArray());
int offset = 0;
for (byte[] array : bytes) {
System.arraycopy(array, 0, valueBytes, offset, array.length);
offset += array.length;
}
cache.put(key, valueBytes);
} }
catch (IOException e) { catch (IOException e) {
throw Throwables.propagate(e); throw Throwables.propagate(e);

15
server/src/test/java/io/druid/client/CachingClusteredClientTest.java
View File

@ -1307,9 +1307,9 @@ public class CachingClusteredClientTest
makeSelectResults(dimensions, metrics, DateTimes.of("2011-01-02"), ImmutableMap.of("a", "c", "rows", 5)), makeSelectResults(dimensions, metrics, DateTimes.of("2011-01-02"), ImmutableMap.of("a", "c", "rows", 5)),
Intervals.of("2011-01-05/2011-01-10"), Intervals.of("2011-01-05/2011-01-10"),
makeSelectResults(dimensions, metrics, DateTimes.of("2011-01-05"), ImmutableMap.of("a", "d", "rows", 5), makeSelectResults(dimensions, metrics, DateTimes.of("2011-01-05"),
DateTimes.of("2011-01-06"), ImmutableMap.of("a", "e", "rows", 6), DateTimes.of("2011-01-06"),
DateTimes.of("2011-01-07"), ImmutableMap.of("a", "f", "rows", 7), DateTimes.of("2011-01-07"), ImmutableMap.of("a", "f", "rows", 7), ImmutableMap.of("a", "ff"),
DateTimes.of("2011-01-08"), ImmutableMap.of("a", "g", "rows", 8), DateTimes.of("2011-01-08"), ImmutableMap.of("a", "g", "rows", 8),
DateTimes.of("2011-01-09"), ImmutableMap.of("a", "h", "rows", 9) DateTimes.of("2011-01-09"), ImmutableMap.of("a", "h", "rows", 9)
), ),
@ -1335,11 +1335,11 @@ public class CachingClusteredClientTest
TestHelper.assertExpectedResults( TestHelper.assertExpectedResults(
makeSelectResults(dimensions, metrics, DateTimes.of("2011-01-01"), ImmutableMap.of("a", "b", "rows", 1), makeSelectResults(dimensions, metrics, DateTimes.of("2011-01-01"), ImmutableMap.of("a", "b", "rows", 1),
DateTimes.of("2011-01-02"), ImmutableMap.of("a", "c", "rows", 5), DateTimes.of("2011-01-02"), ImmutableMap.of("a", "c", "rows", 5),
DateTimes.of("2011-01-05"), ImmutableMap.of("a", "d", "rows", 5), DateTimes.of("2011-01-05"),
DateTimes.of("2011-01-05T01"), ImmutableMap.of("a", "d", "rows", 5), DateTimes.of("2011-01-05T01"), ImmutableMap.of("a", "d", "rows", 5),
DateTimes.of("2011-01-06"), ImmutableMap.of("a", "e", "rows", 6), DateTimes.of("2011-01-06"),
DateTimes.of("2011-01-06T01"), ImmutableMap.of("a", "e", "rows", 6), DateTimes.of("2011-01-06T01"), ImmutableMap.of("a", "e", "rows", 6),
DateTimes.of("2011-01-07"), ImmutableMap.of("a", "f", "rows", 7), DateTimes.of("2011-01-07"), ImmutableMap.of("a", "f", "rows", 7), ImmutableMap.of("a", "ff"),
DateTimes.of("2011-01-07T01"), ImmutableMap.of("a", "f", "rows", 7), DateTimes.of("2011-01-07T01"), ImmutableMap.of("a", "f", "rows", 7),
DateTimes.of("2011-01-08"), ImmutableMap.of("a", "g", "rows", 8), DateTimes.of("2011-01-08"), ImmutableMap.of("a", "g", "rows", 8),
DateTimes.of("2011-01-08T01"), ImmutableMap.of("a", "g", "rows", 8), DateTimes.of("2011-01-08T01"), ImmutableMap.of("a", "g", "rows", 8),
@ -2654,7 +2654,8 @@ public class CachingClusteredClientTest
retVal.add(new Result<>( retVal.add(new Result<>(
timestamp, timestamp,
new SelectResultValue(null, dimensions, metrics, values) new SelectResultValue(ImmutableMap.of(timestamp.toString(), 0),
dimensions, metrics, values)
)); ));
} }
return retVal; return retVal;