fix missing charsets

This commit is contained in:
Xavier Léauté 2014-09-15 12:15:11 -07:00
parent cfa92e8217
commit d4795ce927
30 changed files with 77 additions and 37 deletions

View File

@ -20,6 +20,7 @@
package io.druid.examples.web;
import com.google.api.client.repackaged.com.google.common.base.Throwables;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.io.InputSupplier;
import com.metamx.emitter.EmittingLogger;
@ -55,6 +56,6 @@ public class WebJsonSupplier implements InputSupplier<BufferedReader>
{
URLConnection connection = url.openConnection();
connection.setDoInput(true);
return new BufferedReader(new InputStreamReader(url.openStream()));
return new BufferedReader(new InputStreamReader(url.openStream(), Charsets.UTF_8));
}
}

View File

@ -21,6 +21,7 @@ package io.druid.granularity;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import org.joda.time.Chronology;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
@ -325,7 +326,7 @@ public class PeriodGranularity extends BaseQueryGranularity
@Override
public byte[] cacheKey()
{
return (period.toString() + ":" + chronology.getZone().toString()).getBytes();
return (period.toString() + ":" + chronology.getZone().toString()).getBytes(Charsets.UTF_8);
}
@Override

View File

@ -21,6 +21,7 @@ package io.druid.query.aggregation;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.primitives.Doubles;
import io.druid.segment.ColumnSelectorFactory;
@ -129,7 +130,7 @@ public class DoubleSumAggregatorFactory implements AggregatorFactory
@Override
public byte[] getCacheKey()
{
byte[] fieldNameBytes = fieldName.getBytes();
byte[] fieldNameBytes = fieldName.getBytes(Charsets.UTF_8);
return ByteBuffer.allocate(1 + fieldNameBytes.length).put(CACHE_TYPE_ID).put(fieldNameBytes).array();
}

View File

@ -154,7 +154,7 @@ public class HistogramAggregatorFactory implements AggregatorFactory
@Override
public byte[] getCacheKey()
{
byte[] fieldNameBytes = fieldName.getBytes();
byte[] fieldNameBytes = fieldName.getBytes(Charsets.UTF_8);
return ByteBuffer.allocate(1 + fieldNameBytes.length).put(CACHE_TYPE_ID).put(fieldNameBytes).array();
}

View File

@ -21,6 +21,7 @@ package io.druid.query.aggregation;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.primitives.Longs;
import io.druid.segment.ColumnSelectorFactory;
@ -125,7 +126,7 @@ public class LongSumAggregatorFactory implements AggregatorFactory
@Override
public byte[] getCacheKey()
{
byte[] fieldNameBytes = fieldName.getBytes();
byte[] fieldNameBytes = fieldName.getBytes(Charsets.UTF_8);
return ByteBuffer.allocate(1 + fieldNameBytes.length).put(CACHE_TYPE_ID).put(fieldNameBytes).array();
}

View File

@ -21,6 +21,7 @@ package io.druid.query.aggregation;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.primitives.Doubles;
import io.druid.segment.ColumnSelectorFactory;
@ -126,7 +127,7 @@ public class MaxAggregatorFactory implements AggregatorFactory
@Override
public byte[] getCacheKey()
{
byte[] fieldNameBytes = fieldName.getBytes();
byte[] fieldNameBytes = fieldName.getBytes(Charsets.UTF_8);
return ByteBuffer.allocate(1 + fieldNameBytes.length).put(CACHE_TYPE_ID).put(fieldNameBytes).array();
}

View File

@ -21,6 +21,7 @@ package io.druid.query.aggregation;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.primitives.Doubles;
import io.druid.segment.ColumnSelectorFactory;
@ -126,7 +127,7 @@ public class MinAggregatorFactory implements AggregatorFactory
@Override
public byte[] getCacheKey()
{
byte[] fieldNameBytes = fieldName.getBytes();
byte[] fieldNameBytes = fieldName.getBytes(Charsets.UTF_8);
return ByteBuffer.allocate(1 + fieldNameBytes.length).put(CACHE_TYPE_ID).put(fieldNameBytes).array();
}

View File

@ -21,6 +21,7 @@ package io.druid.query.dimension;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import io.druid.query.extraction.DimExtractionFn;
import java.nio.ByteBuffer;
@ -68,7 +69,7 @@ public class DefaultDimensionSpec implements DimensionSpec
@Override
public byte[] getCacheKey()
{
byte[] dimensionBytes = dimension.getBytes();
byte[] dimensionBytes = dimension.getBytes(Charsets.UTF_8);
return ByteBuffer.allocate(1 + dimensionBytes.length)
.put(CACHE_TYPE_ID)

View File

@ -21,6 +21,7 @@ package io.druid.query.dimension;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import io.druid.query.extraction.DimExtractionFn;
import java.nio.ByteBuffer;
@ -73,7 +74,7 @@ public class ExtractionDimensionSpec implements DimensionSpec
@Override
public byte[] getCacheKey()
{
byte[] dimensionBytes = dimension.getBytes();
byte[] dimensionBytes = dimension.getBytes(Charsets.UTF_8);
byte[] dimExtractionFnBytes = dimExtractionFn.getCacheKey();
return ByteBuffer.allocate(1 + dimensionBytes.length + dimExtractionFnBytes.length)

View File

@ -21,6 +21,7 @@ package io.druid.query.extraction;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import com.google.common.base.Function;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.ContextFactory;
@ -80,7 +81,7 @@ public class JavascriptDimExtractionFn implements DimExtractionFn
@Override
public byte[] getCacheKey()
{
byte[] bytes = function.getBytes();
byte[] bytes = function.getBytes(Charsets.UTF_8);
return ByteBuffer.allocate(1 + bytes.length)
.put(CACHE_TYPE_ID)
.put(bytes)

View File

@ -21,6 +21,7 @@ package io.druid.query.extraction;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import java.nio.ByteBuffer;
import java.util.regex.Matcher;
@ -47,7 +48,7 @@ public class PartialDimExtractionFn implements DimExtractionFn
@Override
public byte[] getCacheKey()
{
byte[] exprBytes = expr.getBytes();
byte[] exprBytes = expr.getBytes(Charsets.UTF_8);
return ByteBuffer.allocate(1 + exprBytes.length)
.put(CACHE_TYPE_ID)
.put(exprBytes)

View File

@ -21,6 +21,7 @@ package io.druid.query.extraction;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import java.nio.ByteBuffer;
import java.util.regex.Matcher;
@ -47,7 +48,7 @@ public class RegexDimExtractionFn implements DimExtractionFn
@Override
public byte[] getCacheKey()
{
byte[] exprBytes = expr.getBytes();
byte[] exprBytes = expr.getBytes(Charsets.UTF_8);
return ByteBuffer.allocate(1 + exprBytes.length)
.put(CACHE_TYPE_ID)
.put(exprBytes)

View File

@ -21,6 +21,7 @@ package io.druid.query.extraction;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import com.ibm.icu.text.SimpleDateFormat;
import java.nio.ByteBuffer;
@ -55,7 +56,7 @@ public class TimeDimExtractionFn implements DimExtractionFn
@Override
public byte[] getCacheKey()
{
byte[] timeFormatBytes = timeFormat.getBytes();
byte[] timeFormatBytes = timeFormat.getBytes(Charsets.UTF_8);
return ByteBuffer.allocate(1 + timeFormatBytes.length)
.put(CACHE_TYPE_ID)
.put(timeFormatBytes)

View File

@ -21,6 +21,7 @@ package io.druid.query.filter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import io.druid.query.extraction.DimExtractionFn;
@ -71,8 +72,8 @@ public class ExtractionDimFilter implements DimFilter
@Override
public byte[] getCacheKey()
{
byte[] dimensionBytes = dimension.getBytes();
byte[] valueBytes = value.getBytes();
byte[] dimensionBytes = dimension.getBytes(Charsets.UTF_8);
byte[] valueBytes = value.getBytes(Charsets.UTF_8);
return ByteBuffer.allocate(1 + dimensionBytes.length + valueBytes.length)
.put(DimFilterCacheHelper.EXTRACTION_CACHE_ID)

View File

@ -21,6 +21,7 @@ package io.druid.query.filter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import java.nio.ByteBuffer;
@ -47,8 +48,8 @@ public class SelectorDimFilter implements DimFilter
@Override
public byte[] getCacheKey()
{
byte[] dimensionBytes = dimension.getBytes();
byte[] valueBytes = (value == null) ? new byte[]{} : value.getBytes();
byte[] dimensionBytes = dimension.getBytes(Charsets.UTF_8);
byte[] valueBytes = (value == null) ? new byte[]{} : value.getBytes(Charsets.UTF_8);
return ByteBuffer.allocate(1 + dimensionBytes.length + valueBytes.length)
.put(DimFilterCacheHelper.SELECTOR_CACHE_ID)

View File

@ -20,6 +20,7 @@ package io.druid.query.filter;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.metamx.collections.spatial.search.Bound;
@ -48,7 +49,7 @@ public class SpatialDimFilter implements DimFilter
@Override
public byte[] getCacheKey()
{
byte[] dimBytes = dimension.getBytes();
byte[] dimBytes = dimension.getBytes(Charsets.UTF_8);
byte[] boundBytes = bound.getCacheKey();
return ByteBuffer.allocate(1 + dimBytes.length + boundBytes.length)

View File

@ -21,6 +21,7 @@ package io.druid.query.groupby.having;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import com.google.common.primitives.Bytes;
import io.druid.data.input.Row;
@ -71,7 +72,7 @@ public class EqualToHavingSpec implements HavingSpec
@Override
public byte[] getCacheKey()
{
final byte[] aggBytes = aggregationName.getBytes();
final byte[] aggBytes = aggregationName.getBytes(Charsets.UTF_8);
final byte[] valBytes = Bytes.toArray(Arrays.asList(value));
return ByteBuffer.allocate(1 + aggBytes.length + valBytes.length)
.put(CACHE_KEY)

View File

@ -21,6 +21,7 @@ package io.druid.query.groupby.having;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import com.google.common.primitives.Bytes;
import io.druid.data.input.Row;
@ -71,7 +72,7 @@ public class GreaterThanHavingSpec implements HavingSpec
@Override
public byte[] getCacheKey()
{
final byte[] aggBytes = aggregationName.getBytes();
final byte[] aggBytes = aggregationName.getBytes(Charsets.UTF_8);
final byte[] valBytes = Bytes.toArray(Arrays.asList(value));
return ByteBuffer.allocate(1 + aggBytes.length + valBytes.length)
.put(CACHE_KEY)

View File

@ -20,6 +20,7 @@
package io.druid.query.groupby.having;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import com.google.common.primitives.Bytes;
import io.druid.data.input.Row;
@ -69,7 +70,7 @@ public class LessThanHavingSpec implements HavingSpec
@Override
public byte[] getCacheKey()
{
final byte[] aggBytes = aggregationName.getBytes();
final byte[] aggBytes = aggregationName.getBytes(Charsets.UTF_8);
final byte[] valBytes = Bytes.toArray(Arrays.asList(value));
return ByteBuffer.allocate(1 + aggBytes.length + valBytes.length)
.put(CACHE_KEY)

View File

@ -21,6 +21,7 @@ package io.druid.query.groupby.orderby;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
@ -182,8 +183,8 @@ public class OrderByColumnSpec
public byte[] getCacheKey()
{
final byte[] dimensionBytes = dimension.getBytes();
final byte[] directionBytes = direction.name().getBytes();
final byte[] dimensionBytes = dimension.getBytes(Charsets.UTF_8);
final byte[] directionBytes = direction.name().getBytes(Charsets.UTF_8);
return ByteBuffer.allocate(dimensionBytes.length + directionBytes.length)
.put(dimensionBytes)

View File

@ -20,6 +20,7 @@
package io.druid.query.search;
import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Charsets;
import com.google.common.base.Function;
import com.google.common.base.Functions;
import com.google.common.base.Joiner;
@ -172,7 +173,7 @@ public class SearchQueryQueryToolChest extends QueryToolChest<Result<SearchResul
int dimensionsBytesSize = 0;
int index = 0;
for (String dimension : dimensions) {
dimensionsBytes[index] = dimension.getBytes();
dimensionsBytes[index] = dimension.getBytes(Charsets.UTF_8);
dimensionsBytesSize += dimensionsBytes[index].length;
++index;
}

View File

@ -21,6 +21,7 @@ package io.druid.query.search.search;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
@ -77,7 +78,7 @@ public class FragmentSearchQuerySpec implements SearchQuerySpec
int valuesBytesSize = 0;
int index = 0;
for (String value : values) {
valuesBytes[index] = value.getBytes();
valuesBytes[index] = value.getBytes(Charsets.UTF_8);
valuesBytesSize += valuesBytes[index].length;
++index;
}

View File

@ -21,6 +21,7 @@ package io.druid.query.search.search;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import java.nio.ByteBuffer;
@ -58,7 +59,7 @@ public class InsensitiveContainsSearchQuerySpec implements SearchQuerySpec
@Override
public byte[] getCacheKey()
{
byte[] valueBytes = value.getBytes();
byte[] valueBytes = value.getBytes(Charsets.UTF_8);
return ByteBuffer.allocate(1 + valueBytes.length)
.put(CACHE_TYPE_ID)

View File

@ -21,6 +21,7 @@ package io.druid.query.select;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import com.google.common.primitives.Ints;
import java.nio.ByteBuffer;
@ -65,7 +66,7 @@ public class PagingSpec
int pagingKeysSize = 0;
int pagingValuesSize = 0;
for (Map.Entry<String, Integer> entry : pagingIdentifiers.entrySet()) {
pagingKeys[index] = entry.getKey().getBytes();
pagingKeys[index] = entry.getKey().getBytes(Charsets.UTF_8);
pagingValues[index] = ByteBuffer.allocate(Ints.BYTES).putInt(entry.getValue()).array();
pagingKeysSize += pagingKeys[index].length;
pagingValuesSize += Ints.BYTES;

View File

@ -21,6 +21,7 @@ package io.druid.query.select;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Charsets;
import com.google.common.base.Function;
import com.google.common.base.Functions;
import com.google.common.base.Joiner;
@ -173,7 +174,7 @@ public class SelectQueryQueryToolChest extends QueryToolChest<Result<SelectResul
int dimensionsBytesSize = 0;
int index = 0;
for (String dimension : dimensions) {
dimensionsBytes[index] = dimension.getBytes();
dimensionsBytes[index] = dimension.getBytes(Charsets.UTF_8);
dimensionsBytesSize += dimensionsBytes[index].length;
++index;
}
@ -187,7 +188,7 @@ public class SelectQueryQueryToolChest extends QueryToolChest<Result<SelectResul
int metricBytesSize = 0;
index = 0;
for (String metric : metrics) {
metricBytes[index] = metric.getBytes();
metricBytes[index] = metric.getBytes(Charsets.UTF_8);
metricBytesSize += metricBytes[index].length;
++index;
}

View File

@ -21,6 +21,7 @@ package io.druid.firehose.rabbitmq;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import com.metamx.common.logger.Logger;
import com.rabbitmq.client.AMQP;
import com.rabbitmq.client.Channel;
@ -227,7 +228,7 @@ public class RabbitMQFirehoseFactory implements FirehoseFactory<StringInputRowPa
return null;
}
return stringParser.parse(new String(delivery.getBody()));
return stringParser.parse(new String(delivery.getBody(), Charsets.UTF_8));
}
@Override

View File

@ -20,6 +20,7 @@
package io.druid.client;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Charsets;
import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import io.druid.client.cache.Cache;
@ -39,7 +40,7 @@ public class CacheUtil
)
{
final Interval segmentQueryInterval = descriptor.getInterval();
final byte[] versionBytes = descriptor.getVersion().getBytes();
final byte[] versionBytes = descriptor.getVersion().getBytes(Charsets.UTF_8);
return new Cache.NamedKey(
segmentIdentifier, ByteBuffer

View File

@ -20,6 +20,7 @@
package io.druid.initialization;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Charsets;
import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@ -74,6 +75,7 @@ import org.eclipse.aether.util.filter.DependencyFilterUtils;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
@ -294,13 +296,17 @@ public class Initialization
}
}
)
, false, Charsets.UTF_8.name())
);
return new DefaultTeslaAether(
config.getLocalRepository(),
remoteRepositories.toArray(new Repository[remoteRepositories.size()])
);
}
catch(UnsupportedEncodingException e) {
// should never happen
throw new IllegalStateException(e);
}
finally {
System.setOut(oldOut);
}

View File

@ -20,6 +20,7 @@
package io.druid.server.log;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Charsets;
import com.google.common.base.Throwables;
import com.metamx.common.concurrent.ScheduledExecutors;
import com.metamx.common.guava.CloseQuietly;
@ -31,8 +32,9 @@ import org.joda.time.Duration;
import org.joda.time.MutableDateTime;
import java.io.File;
import java.io.FileWriter;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.concurrent.Callable;
import java.util.concurrent.ScheduledExecutorService;
@ -47,7 +49,7 @@ public class FileRequestLogger implements RequestLogger
private final Object lock = new Object();
private volatile DateTime currentDay;
private volatile FileWriter fileWriter;
private volatile OutputStreamWriter fileWriter;
public FileRequestLogger(ObjectMapper objectMapper, ScheduledExecutorService exec, File baseDir)
{
@ -66,7 +68,10 @@ public class FileRequestLogger implements RequestLogger
mutableDateTime.setMillisOfDay(0);
currentDay = mutableDateTime.toDateTime();
fileWriter = new FileWriter(new File(baseDir, currentDay.toString("yyyy-MM-dd'.log'")), true);
fileWriter = new OutputStreamWriter(
new FileOutputStream(new File(baseDir, currentDay.toString("yyyy-MM-dd'.log'")), true),
Charsets.UTF_8
);
long nextDay = currentDay.plusDays(1).getMillis();
Duration delay = new Duration(nextDay - new DateTime().getMillis());
@ -84,7 +89,10 @@ public class FileRequestLogger implements RequestLogger
try {
synchronized (lock) {
CloseQuietly.close(fileWriter);
fileWriter = new FileWriter(new File(baseDir, currentDay.toString()), true);
fileWriter = new OutputStreamWriter(
new FileOutputStream(new File(baseDir, currentDay.toString()), true),
Charsets.UTF_8
);
}
}
catch (Exception e) {

View File

@ -145,7 +145,7 @@ public class SQLRunner
final URLConnection urlConnection = url.openConnection();
urlConnection.addRequestProperty("content-type", "application/json");
urlConnection.getOutputStream().write(queryStr.getBytes(Charsets.UTF_8));
BufferedReader stdInput = new BufferedReader(new InputStreamReader(urlConnection.getInputStream()));
BufferedReader stdInput = new BufferedReader(new InputStreamReader(urlConnection.getInputStream(), Charsets.UTF_8));
Object res = objectMapper.readValue(stdInput, typeRef);