From b1b5c9099e335b69641209fcdb9400f78b9220fc Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Tue, 9 Dec 2014 17:03:01 -0800 Subject: [PATCH] Update all String conversions to and from byte[] to use the java-util StringUtils functions * Speedup of GroupBy with javaScript filters by ~10% * Requires https://github.com/metamx/java-util/pull/15 --- .../histogram/ApproximateHistogramAggregatorFactory.java | 5 +++-- .../ApproximateHistogramFoldingAggregatorFactory.java | 3 ++- .../druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java | 3 ++- .../src/main/java/io/druid/indexer/SortableBytes.java | 5 +++-- .../src/test/java/io/druid/indexer/SortableBytesTest.java | 5 +++-- .../overlord/autoscaling/ec2/StringEC2UserData.java | 3 ++- .../io/druid/indexing/overlord/WorkerSetupDataTest.java | 5 +++-- pom.xml | 2 +- .../main/java/io/druid/granularity/PeriodGranularity.java | 3 ++- .../query/aggregation/DoubleSumAggregatorFactory.java | 3 ++- .../query/aggregation/HistogramAggregatorFactory.java | 5 +++-- .../query/aggregation/JavaScriptAggregatorFactory.java | 5 +++-- .../druid/query/aggregation/LongSumAggregatorFactory.java | 3 ++- .../io/druid/query/aggregation/MaxAggregatorFactory.java | 3 ++- .../io/druid/query/aggregation/MinAggregatorFactory.java | 3 ++- .../cardinality/CardinalityAggregatorFactory.java | 5 +++-- .../hyperloglog/HyperUniquesAggregatorFactory.java | 5 +++-- .../query/aggregation/hyperloglog/HyperUniquesSerde.java | 3 ++- .../io/druid/query/dimension/DefaultDimensionSpec.java | 3 ++- .../io/druid/query/dimension/ExtractionDimensionSpec.java | 3 ++- .../druid/query/extraction/JavascriptDimExtractionFn.java | 3 ++- .../io/druid/query/extraction/PartialDimExtractionFn.java | 3 ++- .../io/druid/query/extraction/RegexDimExtractionFn.java | 3 ++- .../io/druid/query/extraction/TimeDimExtractionFn.java | 3 ++- .../java/io/druid/query/filter/ExtractionDimFilter.java | 5 +++-- .../java/io/druid/query/filter/JavaScriptDimFilter.java | 5 +++-- .../main/java/io/druid/query/filter/RegexDimFilter.java | 5 +++-- .../java/io/druid/query/filter/SearchQueryDimFilter.java | 3 ++- .../java/io/druid/query/filter/SelectorDimFilter.java | 5 +++-- .../main/java/io/druid/query/filter/SpatialDimFilter.java | 3 ++- .../io/druid/query/groupby/having/EqualToHavingSpec.java | 3 ++- .../druid/query/groupby/having/GreaterThanHavingSpec.java | 3 ++- .../io/druid/query/groupby/having/LessThanHavingSpec.java | 3 ++- .../io/druid/query/groupby/orderby/OrderByColumnSpec.java | 5 +++-- .../java/io/druid/query/metadata/SegmentAnalyzer.java | 3 ++- .../query/metadata/metadata/ListColumnIncluderator.java | 3 ++- .../io/druid/query/search/SearchQueryQueryToolChest.java | 3 ++- .../query/search/search/FragmentSearchQuerySpec.java | 3 ++- .../search/search/InsensitiveContainsSearchQuerySpec.java | 3 ++- .../src/main/java/io/druid/query/select/PagingSpec.java | 3 ++- .../io/druid/query/select/SelectQueryQueryToolChest.java | 5 +++-- .../io/druid/query/timeboundary/TimeBoundaryQuery.java | 3 ++- .../io/druid/query/topn/AlphaNumericTopNMetricSpec.java | 3 ++- .../io/druid/query/topn/LexicographicTopNMetricSpec.java | 8 ++++++-- .../java/io/druid/query/topn/NumericTopNMetricSpec.java | 2 +- .../main/java/io/druid/segment/data/GenericIndexed.java | 8 ++++---- server/src/main/java/io/druid/client/CacheUtil.java | 2 +- .../main/java/io/druid/client/ServerInventoryView.java | 2 ++ server/src/main/java/io/druid/client/cache/Cache.java | 3 ++- .../main/java/io/druid/initialization/Initialization.java | 3 ++- .../druid/metadata/SQLMetadataStorageActionHandler.java | 5 +++-- server/src/main/java/io/druid/server/QueryResource.java | 1 + server/src/main/java/io/druid/server/sql/SQLRunner.java | 3 ++- 53 files changed, 123 insertions(+), 70 deletions(-) diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java b/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java index fbe88a26e1f..ac2dcba453e 100644 --- a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java +++ b/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramAggregatorFactory.java @@ -27,6 +27,7 @@ import com.google.common.base.Charsets; import com.google.common.base.Preconditions; import com.google.common.primitives.Floats; import com.google.common.primitives.Ints; +import com.metamx.common.StringUtils; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.Aggregators; @@ -153,7 +154,7 @@ public class ApproximateHistogramAggregatorFactory implements AggregatorFactory return ah; } else if (object instanceof String) { - byte[] bytes = Base64.decodeBase64(((String) object).getBytes(Charsets.UTF_8)); + byte[] bytes = Base64.decodeBase64(StringUtils.toUtf8((String) object)); final ApproximateHistogram ah = ApproximateHistogram.fromBytes(bytes); ah.setLowerLimit(lowerLimit); ah.setUpperLimit(upperLimit); @@ -216,7 +217,7 @@ public class ApproximateHistogramAggregatorFactory implements AggregatorFactory @Override public byte[] getCacheKey() { - byte[] fieldNameBytes = fieldName.getBytes(Charsets.UTF_8); + byte[] fieldNameBytes = StringUtils.toUtf8(fieldName); return ByteBuffer.allocate(1 + fieldNameBytes.length + Ints.BYTES * 2 + Floats.BYTES * 2) .put(CACHE_TYPE_ID) .put(fieldNameBytes) diff --git a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java b/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java index 50a9e6ba973..3dee83ec09e 100644 --- a/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java +++ b/extensions/histogram/src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingAggregatorFactory.java @@ -26,6 +26,7 @@ import com.google.common.base.Charsets; import com.google.common.primitives.Floats; import com.google.common.primitives.Ints; import com.metamx.common.IAE; +import com.metamx.common.StringUtils; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.BufferAggregator; @@ -139,7 +140,7 @@ public class ApproximateHistogramFoldingAggregatorFactory extends ApproximateHis @Override public byte[] getCacheKey() { - byte[] fieldNameBytes = fieldName.getBytes(Charsets.UTF_8); + byte[] fieldNameBytes = StringUtils.toUtf8(fieldName); return ByteBuffer.allocate(1 + fieldNameBytes.length + Ints.BYTES * 2 + Floats.BYTES * 2) .put(CACHE_TYPE_ID) .put(fieldNameBytes) diff --git a/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java b/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java index c44032f4370..4b7c250f66a 100644 --- a/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java +++ b/extensions/rabbitmq/src/main/java/io/druid/firehose/rabbitmq/RabbitMQFirehoseFactory.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.metamx.common.logger.Logger; +import com.metamx.common.StringUtils; import com.rabbitmq.client.AMQP; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; @@ -234,7 +235,7 @@ public class RabbitMQFirehoseFactory implements FirehoseFactory @@ -75,7 +76,7 @@ public class StringEC2UserData implements EC2UserData } else { finalData = data; } - return Base64.encodeBase64String(finalData.getBytes(Charsets.UTF_8)); + return Base64.encodeBase64String(StringUtils.toUtf8(finalData)); } @Override diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/WorkerSetupDataTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/WorkerSetupDataTest.java index 10f2495c592..8e21266d800 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/WorkerSetupDataTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/WorkerSetupDataTest.java @@ -20,6 +20,7 @@ package io.druid.indexing.overlord; import com.google.common.base.Charsets; +import com.metamx.common.StringUtils; import io.druid.indexing.common.TestUtils; import io.druid.indexing.overlord.autoscaling.ec2.EC2UserData; import io.druid.indexing.overlord.autoscaling.ec2.GalaxyEC2UserData; @@ -52,11 +53,11 @@ public class WorkerSetupDataTest Assert.assertEquals("hey :ver:", userData.getData()); Assert.assertEquals("1234", userData.getVersion()); Assert.assertEquals( - Base64.encodeBase64String("hey 1234".getBytes(Charsets.UTF_8)), + Base64.encodeBase64String(StringUtils.toUtf8("hey 1234")), userData.getUserDataBase64() ); Assert.assertEquals( - Base64.encodeBase64String("hey xyz".getBytes(Charsets.UTF_8)), + Base64.encodeBase64String(StringUtils.toUtf8("hey xyz")), userData.withVersion("xyz").getUserDataBase64() ); } diff --git a/pom.xml b/pom.xml index b82343b5ff1..cae8f16c79a 100644 --- a/pom.xml +++ b/pom.xml @@ -40,7 +40,7 @@ UTF-8 - 0.26.9 + 0.26.11 2.7.0 9.2.5.v20141112 0.3.1 diff --git a/processing/src/main/java/io/druid/granularity/PeriodGranularity.java b/processing/src/main/java/io/druid/granularity/PeriodGranularity.java index c3e4feff90d..454bbd28bd8 100644 --- a/processing/src/main/java/io/druid/granularity/PeriodGranularity.java +++ b/processing/src/main/java/io/druid/granularity/PeriodGranularity.java @@ -22,6 +22,7 @@ package io.druid.granularity; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; +import com.metamx.common.StringUtils; import org.joda.time.Chronology; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; @@ -326,7 +327,7 @@ public class PeriodGranularity extends BaseQueryGranularity @Override public byte[] cacheKey() { - return (period.toString() + ":" + chronology.getZone().toString() + ":" + origin).getBytes(Charsets.UTF_8); + return StringUtils.toUtf8(period.toString() + ":" + chronology.getZone().toString() + ":" + origin); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java index 713022efade..067b59e5211 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; import com.google.common.primitives.Doubles; +import com.metamx.common.StringUtils; import io.druid.segment.ColumnSelectorFactory; import java.nio.ByteBuffer; @@ -130,7 +131,7 @@ public class DoubleSumAggregatorFactory implements AggregatorFactory @Override public byte[] getCacheKey() { - byte[] fieldNameBytes = fieldName.getBytes(Charsets.UTF_8); + byte[] fieldNameBytes = StringUtils.toUtf8(fieldName); return ByteBuffer.allocate(1 + fieldNameBytes.length).put(CACHE_TYPE_ID).put(fieldNameBytes).array(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/HistogramAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/HistogramAggregatorFactory.java index 5a99ac9d18c..5a265662c49 100644 --- a/processing/src/main/java/io/druid/query/aggregation/HistogramAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/HistogramAggregatorFactory.java @@ -26,6 +26,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.primitives.Floats; import com.google.common.primitives.Longs; +import com.metamx.common.StringUtils; import io.druid.segment.ColumnSelectorFactory; import org.apache.commons.codec.binary.Base64; @@ -114,7 +115,7 @@ public class HistogramAggregatorFactory implements AggregatorFactory } else if (object instanceof ByteBuffer) { return Histogram.fromBytes((ByteBuffer) object); } else if (object instanceof String) { - byte[] bytes = Base64.decodeBase64(((String) object).getBytes(Charsets.UTF_8)); + byte[] bytes = Base64.decodeBase64(StringUtils.toUtf8((String) object)); return Histogram.fromBytes(bytes); } return object; @@ -154,7 +155,7 @@ public class HistogramAggregatorFactory implements AggregatorFactory @Override public byte[] getCacheKey() { - byte[] fieldNameBytes = fieldName.getBytes(Charsets.UTF_8); + byte[] fieldNameBytes = StringUtils.toUtf8(fieldName); return ByteBuffer.allocate(1 + fieldNameBytes.length).put(CACHE_TYPE_ID).put(fieldNameBytes).array(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregatorFactory.java index 3898e50d239..8873d817c0c 100644 --- a/processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregatorFactory.java @@ -26,6 +26,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.primitives.Doubles; +import com.metamx.common.StringUtils; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ObjectColumnSelector; import org.mozilla.javascript.Context; @@ -211,8 +212,8 @@ public class JavaScriptAggregatorFactory implements AggregatorFactory { try { MessageDigest md = MessageDigest.getInstance("SHA-1"); - byte[] fieldNameBytes = Joiner.on(",").join(fieldNames).getBytes(Charsets.UTF_8); - byte[] sha1 = md.digest((fnAggregate + fnReset + fnCombine).getBytes(Charsets.UTF_8)); + byte[] fieldNameBytes = StringUtils.toUtf8(Joiner.on(",").join(fieldNames)); + byte[] sha1 = md.digest(StringUtils.toUtf8(fnAggregate + fnReset + fnCombine)); return ByteBuffer.allocate(1 + fieldNameBytes.length + sha1.length) .put(CACHE_TYPE_ID) diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java index cc7d218fc30..8725b357d4c 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; import com.google.common.primitives.Longs; +import com.metamx.common.StringUtils; import io.druid.segment.ColumnSelectorFactory; import java.nio.ByteBuffer; @@ -126,7 +127,7 @@ public class LongSumAggregatorFactory implements AggregatorFactory @Override public byte[] getCacheKey() { - byte[] fieldNameBytes = fieldName.getBytes(Charsets.UTF_8); + byte[] fieldNameBytes = StringUtils.toUtf8(fieldName); return ByteBuffer.allocate(1 + fieldNameBytes.length).put(CACHE_TYPE_ID).put(fieldNameBytes).array(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/MaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/MaxAggregatorFactory.java index 23c8b066c25..a2fb5dc22de 100644 --- a/processing/src/main/java/io/druid/query/aggregation/MaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/MaxAggregatorFactory.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; import com.google.common.primitives.Doubles; +import com.metamx.common.StringUtils; import io.druid.segment.ColumnSelectorFactory; import java.nio.ByteBuffer; @@ -127,7 +128,7 @@ public class MaxAggregatorFactory implements AggregatorFactory @Override public byte[] getCacheKey() { - byte[] fieldNameBytes = fieldName.getBytes(Charsets.UTF_8); + byte[] fieldNameBytes = StringUtils.toUtf8(fieldName); return ByteBuffer.allocate(1 + fieldNameBytes.length).put(CACHE_TYPE_ID).put(fieldNameBytes).array(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/MinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/MinAggregatorFactory.java index 859ed2b23cd..175cf01ab9b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/MinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/MinAggregatorFactory.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; import com.google.common.primitives.Doubles; +import com.metamx.common.StringUtils; import io.druid.segment.ColumnSelectorFactory; import java.nio.ByteBuffer; @@ -127,7 +128,7 @@ public class MinAggregatorFactory implements AggregatorFactory @Override public byte[] getCacheKey() { - byte[] fieldNameBytes = fieldName.getBytes(Charsets.UTF_8); + byte[] fieldNameBytes = StringUtils.toUtf8(fieldName); return ByteBuffer.allocate(1 + fieldNameBytes.length).put(CACHE_TYPE_ID).put(fieldNameBytes).array(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java index 368ebbc6f2d..7191fc64131 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java @@ -27,6 +27,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Predicates; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import com.metamx.common.StringUtils; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.Aggregators; @@ -171,7 +172,7 @@ public class CardinalityAggregatorFactory implements AggregatorFactory return HyperLogLogCollector.makeCollector((ByteBuffer) object); } else if (object instanceof String) { return HyperLogLogCollector.makeCollector( - ByteBuffer.wrap(Base64.decodeBase64(((String) object).getBytes(Charsets.UTF_8))) + ByteBuffer.wrap(Base64.decodeBase64(StringUtils.toUtf8((String) object))) ); } return object; @@ -212,7 +213,7 @@ public class CardinalityAggregatorFactory implements AggregatorFactory @Override public byte[] getCacheKey() { - byte[] fieldNameBytes = Joiner.on("\u0001").join(fieldNames).getBytes(Charsets.UTF_8); + byte[] fieldNameBytes = StringUtils.toUtf8(Joiner.on("\u0001").join(fieldNames)); return ByteBuffer.allocate(2 + fieldNameBytes.length) .put(CACHE_TYPE_ID) diff --git a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java index 4c137ffeb70..0f68989cce9 100644 --- a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.metamx.common.IAE; +import com.metamx.common.StringUtils; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.Aggregators; @@ -148,7 +149,7 @@ public class HyperUniquesAggregatorFactory implements AggregatorFactory return HyperLogLogCollector.makeCollector((ByteBuffer) object); } else if (object instanceof String) { return HyperLogLogCollector.makeCollector( - ByteBuffer.wrap(Base64.decodeBase64(((String) object).getBytes(Charsets.UTF_8))) + ByteBuffer.wrap(Base64.decodeBase64(StringUtils.toUtf8((String) object))) ); } return object; @@ -183,7 +184,7 @@ public class HyperUniquesAggregatorFactory implements AggregatorFactory @Override public byte[] getCacheKey() { - byte[] fieldNameBytes = fieldName.getBytes(Charsets.UTF_8); + byte[] fieldNameBytes = StringUtils.toUtf8(fieldName); return ByteBuffer.allocate(1 + fieldNameBytes.length).put(CACHE_TYPE_ID).put(fieldNameBytes).array(); } diff --git a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java index 486616b5eca..9c4cfcf17df 100644 --- a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java +++ b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java @@ -22,6 +22,7 @@ package io.druid.query.aggregation.hyperloglog; import com.google.common.base.Charsets; import com.google.common.collect.Ordering; import com.google.common.hash.HashFunction; +import com.metamx.common.StringUtils; import io.druid.data.input.InputRow; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.GenericIndexed; @@ -93,7 +94,7 @@ public class HyperUniquesSerde extends ComplexMetricSerde for (String dimensionValue : dimValues) { collector.add( - hashFn.hashBytes(dimensionValue.getBytes(Charsets.UTF_8)).asBytes() + hashFn.hashBytes(StringUtils.toUtf8(dimensionValue)).asBytes() ); } return collector; diff --git a/processing/src/main/java/io/druid/query/dimension/DefaultDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/DefaultDimensionSpec.java index cb44fe59c6c..1c4dc46236d 100644 --- a/processing/src/main/java/io/druid/query/dimension/DefaultDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/DefaultDimensionSpec.java @@ -22,6 +22,7 @@ package io.druid.query.dimension; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; +import com.metamx.common.StringUtils; import io.druid.query.extraction.DimExtractionFn; import java.nio.ByteBuffer; @@ -69,7 +70,7 @@ public class DefaultDimensionSpec implements DimensionSpec @Override public byte[] getCacheKey() { - byte[] dimensionBytes = dimension.getBytes(Charsets.UTF_8); + byte[] dimensionBytes = StringUtils.toUtf8(dimension); return ByteBuffer.allocate(1 + dimensionBytes.length) .put(CACHE_TYPE_ID) diff --git a/processing/src/main/java/io/druid/query/dimension/ExtractionDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/ExtractionDimensionSpec.java index c692f1399fb..2d9c45b31e6 100644 --- a/processing/src/main/java/io/druid/query/dimension/ExtractionDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/ExtractionDimensionSpec.java @@ -22,6 +22,7 @@ package io.druid.query.dimension; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; +import com.metamx.common.StringUtils; import io.druid.query.extraction.DimExtractionFn; import java.nio.ByteBuffer; @@ -74,7 +75,7 @@ public class ExtractionDimensionSpec implements DimensionSpec @Override public byte[] getCacheKey() { - byte[] dimensionBytes = dimension.getBytes(Charsets.UTF_8); + byte[] dimensionBytes = StringUtils.toUtf8(dimension); byte[] dimExtractionFnBytes = dimExtractionFn.getCacheKey(); return ByteBuffer.allocate(1 + dimensionBytes.length + dimExtractionFnBytes.length) diff --git a/processing/src/main/java/io/druid/query/extraction/JavascriptDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/JavascriptDimExtractionFn.java index e8099e643a4..7d69791fef4 100644 --- a/processing/src/main/java/io/druid/query/extraction/JavascriptDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/JavascriptDimExtractionFn.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.base.Function; +import com.metamx.common.StringUtils; import org.mozilla.javascript.Context; import org.mozilla.javascript.ContextFactory; import org.mozilla.javascript.ScriptableObject; @@ -81,7 +82,7 @@ public class JavascriptDimExtractionFn implements DimExtractionFn @Override public byte[] getCacheKey() { - byte[] bytes = function.getBytes(Charsets.UTF_8); + byte[] bytes = StringUtils.toUtf8(function); return ByteBuffer.allocate(1 + bytes.length) .put(CACHE_TYPE_ID) .put(bytes) diff --git a/processing/src/main/java/io/druid/query/extraction/PartialDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/PartialDimExtractionFn.java index 34c026f3db3..857994e7643 100644 --- a/processing/src/main/java/io/druid/query/extraction/PartialDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/PartialDimExtractionFn.java @@ -22,6 +22,7 @@ package io.druid.query.extraction; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; +import com.metamx.common.StringUtils; import java.nio.ByteBuffer; import java.util.regex.Matcher; @@ -48,7 +49,7 @@ public class PartialDimExtractionFn implements DimExtractionFn @Override public byte[] getCacheKey() { - byte[] exprBytes = expr.getBytes(Charsets.UTF_8); + byte[] exprBytes = StringUtils.toUtf8(expr); return ByteBuffer.allocate(1 + exprBytes.length) .put(CACHE_TYPE_ID) .put(exprBytes) diff --git a/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java index 636507c1273..2621c9c9f75 100644 --- a/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java @@ -22,6 +22,7 @@ package io.druid.query.extraction; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; +import com.metamx.common.StringUtils; import java.nio.ByteBuffer; import java.util.regex.Matcher; @@ -48,7 +49,7 @@ public class RegexDimExtractionFn implements DimExtractionFn @Override public byte[] getCacheKey() { - byte[] exprBytes = expr.getBytes(Charsets.UTF_8); + byte[] exprBytes = StringUtils.toUtf8(expr); return ByteBuffer.allocate(1 + exprBytes.length) .put(CACHE_TYPE_ID) .put(exprBytes) diff --git a/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java index 25c7bde0cfa..d8e6cbae322 100644 --- a/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.ibm.icu.text.SimpleDateFormat; +import com.metamx.common.StringUtils; import java.nio.ByteBuffer; import java.text.ParseException; @@ -56,7 +57,7 @@ public class TimeDimExtractionFn implements DimExtractionFn @Override public byte[] getCacheKey() { - byte[] timeFormatBytes = timeFormat.getBytes(Charsets.UTF_8); + byte[] timeFormatBytes = StringUtils.toUtf8(timeFormat); return ByteBuffer.allocate(1 + timeFormatBytes.length) .put(CACHE_TYPE_ID) .put(timeFormatBytes) diff --git a/processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java b/processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java index 65fa0e4494c..2b3121f5f38 100644 --- a/processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; +import com.metamx.common.StringUtils; import io.druid.query.extraction.DimExtractionFn; import java.nio.ByteBuffer; @@ -72,8 +73,8 @@ public class ExtractionDimFilter implements DimFilter @Override public byte[] getCacheKey() { - byte[] dimensionBytes = dimension.getBytes(Charsets.UTF_8); - byte[] valueBytes = value.getBytes(Charsets.UTF_8); + byte[] dimensionBytes = StringUtils.toUtf8(dimension); + byte[] valueBytes = StringUtils.toUtf8(value); return ByteBuffer.allocate(1 + dimensionBytes.length + valueBytes.length) .put(DimFilterCacheHelper.EXTRACTION_CACHE_ID) diff --git a/processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java b/processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java index 1f8740562a3..69cf75ec02d 100644 --- a/processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; +import com.metamx.common.StringUtils; import java.nio.ByteBuffer; @@ -58,8 +59,8 @@ public class JavaScriptDimFilter implements DimFilter @Override public byte[] getCacheKey() { - final byte[] dimensionBytes = dimension.getBytes(Charsets.UTF_8); - final byte[] functionBytes = function.getBytes(Charsets.UTF_8); + final byte[] dimensionBytes = StringUtils.toUtf8(dimension); + final byte[] functionBytes = StringUtils.toUtf8(function); return ByteBuffer.allocate(1 + dimensionBytes.length + functionBytes.length) .put(DimFilterCacheHelper.JAVASCRIPT_CACHE_ID) diff --git a/processing/src/main/java/io/druid/query/filter/RegexDimFilter.java b/processing/src/main/java/io/druid/query/filter/RegexDimFilter.java index 9a327dde3b5..1c03cc1a2b2 100644 --- a/processing/src/main/java/io/druid/query/filter/RegexDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/RegexDimFilter.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; +import com.metamx.common.StringUtils; import java.nio.ByteBuffer; @@ -60,8 +61,8 @@ public class RegexDimFilter implements DimFilter @Override public byte[] getCacheKey() { - final byte[] dimensionBytes = dimension.getBytes(Charsets.UTF_8); - final byte[] patternBytes = pattern.getBytes(Charsets.UTF_8); + final byte[] dimensionBytes = StringUtils.toUtf8(dimension); + final byte[] patternBytes = StringUtils.toUtf8(pattern); return ByteBuffer.allocate(1 + dimensionBytes.length + patternBytes.length) .put(DimFilterCacheHelper.REGEX_CACHE_ID) diff --git a/processing/src/main/java/io/druid/query/filter/SearchQueryDimFilter.java b/processing/src/main/java/io/druid/query/filter/SearchQueryDimFilter.java index c6d09c0a55c..5548ae7a36d 100644 --- a/processing/src/main/java/io/druid/query/filter/SearchQueryDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SearchQueryDimFilter.java @@ -22,6 +22,7 @@ package io.druid.query.filter; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; +import com.metamx.common.StringUtils; import io.druid.query.search.search.SearchQuerySpec; import java.nio.ByteBuffer; @@ -60,7 +61,7 @@ public class SearchQueryDimFilter implements DimFilter @Override public byte[] getCacheKey() { - final byte[] dimensionBytes = dimension.getBytes(Charsets.UTF_8); + final byte[] dimensionBytes = StringUtils.toUtf8(dimension); final byte[] queryBytes = query.getCacheKey(); return ByteBuffer.allocate(1 + dimensionBytes.length + queryBytes.length) diff --git a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index 414b475d1fe..c7a0d99a701 100644 --- a/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; +import com.metamx.common.StringUtils; import java.nio.ByteBuffer; @@ -48,8 +49,8 @@ public class SelectorDimFilter implements DimFilter @Override public byte[] getCacheKey() { - byte[] dimensionBytes = dimension.getBytes(Charsets.UTF_8); - byte[] valueBytes = (value == null) ? new byte[]{} : value.getBytes(Charsets.UTF_8); + byte[] dimensionBytes = StringUtils.toUtf8(dimension); + byte[] valueBytes = (value == null) ? new byte[]{} : StringUtils.toUtf8(value); return ByteBuffer.allocate(1 + dimensionBytes.length + valueBytes.length) .put(DimFilterCacheHelper.SELECTOR_CACHE_ID) diff --git a/processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java b/processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java index 7148ebf55e7..3a66e0157d6 100644 --- a/processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; import com.metamx.collections.spatial.search.Bound; +import com.metamx.common.StringUtils; import java.nio.ByteBuffer; @@ -49,7 +50,7 @@ public class SpatialDimFilter implements DimFilter @Override public byte[] getCacheKey() { - byte[] dimBytes = dimension.getBytes(Charsets.UTF_8); + byte[] dimBytes = StringUtils.toUtf8(dimension); byte[] boundBytes = bound.getCacheKey(); return ByteBuffer.allocate(1 + dimBytes.length + boundBytes.length) diff --git a/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java index b0389edfca7..20d821b9da0 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.primitives.Bytes; +import com.metamx.common.StringUtils; import io.druid.data.input.Row; import java.nio.ByteBuffer; @@ -72,7 +73,7 @@ public class EqualToHavingSpec implements HavingSpec @Override public byte[] getCacheKey() { - final byte[] aggBytes = aggregationName.getBytes(Charsets.UTF_8); + final byte[] aggBytes = StringUtils.toUtf8(aggregationName); final byte[] valBytes = Bytes.toArray(Arrays.asList(value)); return ByteBuffer.allocate(1 + aggBytes.length + valBytes.length) .put(CACHE_KEY) diff --git a/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java index 19349a2098b..107ea05f89c 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.primitives.Bytes; +import com.metamx.common.StringUtils; import io.druid.data.input.Row; import java.nio.ByteBuffer; @@ -72,7 +73,7 @@ public class GreaterThanHavingSpec implements HavingSpec @Override public byte[] getCacheKey() { - final byte[] aggBytes = aggregationName.getBytes(Charsets.UTF_8); + final byte[] aggBytes = StringUtils.toUtf8(aggregationName); final byte[] valBytes = Bytes.toArray(Arrays.asList(value)); return ByteBuffer.allocate(1 + aggBytes.length + valBytes.length) .put(CACHE_KEY) diff --git a/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java index 284ddc7042c..f31f3b008f4 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java @@ -22,6 +22,7 @@ package io.druid.query.groupby.having; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.primitives.Bytes; +import com.metamx.common.StringUtils; import io.druid.data.input.Row; import java.nio.ByteBuffer; @@ -70,7 +71,7 @@ public class LessThanHavingSpec implements HavingSpec @Override public byte[] getCacheKey() { - final byte[] aggBytes = aggregationName.getBytes(Charsets.UTF_8); + final byte[] aggBytes = StringUtils.toUtf8(aggregationName); final byte[] valBytes = Bytes.toArray(Arrays.asList(value)); return ByteBuffer.allocate(1 + aggBytes.length + valBytes.length) .put(CACHE_KEY) diff --git a/processing/src/main/java/io/druid/query/groupby/orderby/OrderByColumnSpec.java b/processing/src/main/java/io/druid/query/groupby/orderby/OrderByColumnSpec.java index 2c471d62507..fec5df82f4d 100644 --- a/processing/src/main/java/io/druid/query/groupby/orderby/OrderByColumnSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/orderby/OrderByColumnSpec.java @@ -28,6 +28,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.metamx.common.IAE; import com.metamx.common.ISE; +import com.metamx.common.StringUtils; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -183,8 +184,8 @@ public class OrderByColumnSpec public byte[] getCacheKey() { - final byte[] dimensionBytes = dimension.getBytes(Charsets.UTF_8); - final byte[] directionBytes = direction.name().getBytes(Charsets.UTF_8); + final byte[] dimensionBytes = StringUtils.toUtf8(dimension); + final byte[] directionBytes = StringUtils.toUtf8(direction.name()); return ByteBuffer.allocate(dimensionBytes.length + directionBytes.length) .put(dimensionBytes) diff --git a/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java b/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java index 8f51d95a8f9..26859d81209 100644 --- a/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java +++ b/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java @@ -25,6 +25,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.common.primitives.Longs; import com.metamx.common.logger.Logger; +import com.metamx.common.StringUtils; import io.druid.query.metadata.metadata.ColumnAnalysis; import io.druid.segment.QueryableIndex; import io.druid.segment.column.BitmapIndex; @@ -122,7 +123,7 @@ public class SegmentAnalyzer String value = bitmapIndex.getValue(i); if (value != null) { - size += value.getBytes(Charsets.UTF_8).length * bitmapIndex.getBitmap(value).size(); + size += StringUtils.toUtf8(value).length * bitmapIndex.getBitmap(value).size(); } } diff --git a/processing/src/main/java/io/druid/query/metadata/metadata/ListColumnIncluderator.java b/processing/src/main/java/io/druid/query/metadata/metadata/ListColumnIncluderator.java index 89b5cf8dd6b..299fbc48bb4 100644 --- a/processing/src/main/java/io/druid/query/metadata/metadata/ListColumnIncluderator.java +++ b/processing/src/main/java/io/druid/query/metadata/metadata/ListColumnIncluderator.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import com.metamx.common.StringUtils; import java.nio.ByteBuffer; import java.util.Collections; @@ -64,7 +65,7 @@ public class ListColumnIncluderator implements ColumnIncluderator List columns = Lists.newArrayListWithExpectedSize(this.columns.size()); for (String column : this.columns) { - final byte[] bytes = column.getBytes(Charsets.UTF_8); + final byte[] bytes = StringUtils.toUtf8(column); columns.add(bytes); size += bytes.length; } diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java index f4c38c87507..130072ca880 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java @@ -35,6 +35,7 @@ import com.metamx.common.guava.MergeSequence; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import com.metamx.common.guava.nary.BinaryFn; +import com.metamx.common.StringUtils; import com.metamx.emitter.service.ServiceMetricEvent; import io.druid.collections.OrderedMergeSequence; import io.druid.query.CacheStrategy; @@ -159,7 +160,7 @@ public class SearchQueryQueryToolChest extends QueryToolChest entry : pagingIdentifiers.entrySet()) { - pagingKeys[index] = entry.getKey().getBytes(Charsets.UTF_8); + pagingKeys[index] = StringUtils.toUtf8(entry.getKey()); pagingValues[index] = ByteBuffer.allocate(Ints.BYTES).putInt(entry.getValue()).array(); pagingKeysSize += pagingKeys[index].length; pagingValuesSize += Ints.BYTES; diff --git a/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java index f214d2441b1..e5f72a9b68d 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java @@ -30,6 +30,7 @@ import com.google.inject.Inject; import com.metamx.common.guava.MergeSequence; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.nary.BinaryFn; +import com.metamx.common.StringUtils; import com.metamx.emitter.service.ServiceMetricEvent; import io.druid.collections.OrderedMergeSequence; import io.druid.granularity.QueryGranularity; @@ -160,7 +161,7 @@ public class SelectQueryQueryToolChest extends QueryToolChest public byte[] getCacheKey() { - final byte[] boundBytes = bound.getBytes(Charsets.UTF_8); + final byte[] boundBytes = StringUtils.toUtf8(bound); return ByteBuffer.allocate(1 + boundBytes.length) .put(CACHE_TYPE_ID) .put(boundBytes) diff --git a/processing/src/main/java/io/druid/query/topn/AlphaNumericTopNMetricSpec.java b/processing/src/main/java/io/druid/query/topn/AlphaNumericTopNMetricSpec.java index 008b8dccd76..eee6de75dae 100644 --- a/processing/src/main/java/io/druid/query/topn/AlphaNumericTopNMetricSpec.java +++ b/processing/src/main/java/io/druid/query/topn/AlphaNumericTopNMetricSpec.java @@ -23,6 +23,7 @@ import com.davekoelle.alphanum.AlphanumComparator; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; +import com.metamx.common.StringUtils; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; @@ -54,7 +55,7 @@ public class AlphaNumericTopNMetricSpec extends LexicographicTopNMetricSpec @Override public byte[] getCacheKey() { - byte[] previousStopBytes = previousStop.getBytes(Charsets.UTF_8); + byte[] previousStopBytes = StringUtils.toUtf8(previousStop); return ByteBuffer.allocate(1 + previousStopBytes.length) .put(CACHE_TYPE_ID) diff --git a/processing/src/main/java/io/druid/query/topn/LexicographicTopNMetricSpec.java b/processing/src/main/java/io/druid/query/topn/LexicographicTopNMetricSpec.java index a23dd2f186f..3c1d4afb29d 100644 --- a/processing/src/main/java/io/druid/query/topn/LexicographicTopNMetricSpec.java +++ b/processing/src/main/java/io/druid/query/topn/LexicographicTopNMetricSpec.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.primitives.UnsignedBytes; +import com.metamx.common.StringUtils; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; import io.druid.query.dimension.DimensionSpec; @@ -50,7 +51,10 @@ public class LexicographicTopNMetricSpec implements TopNMetricSpec if (s2 == null) { return 1; } - return UnsignedBytes.lexicographicalComparator().compare(s.getBytes(Charsets.UTF_8), s2.getBytes(Charsets.UTF_8)); + return UnsignedBytes.lexicographicalComparator().compare( + StringUtils.toUtf8(s), + StringUtils.toUtf8(s2) + ); } }; @@ -105,7 +109,7 @@ public class LexicographicTopNMetricSpec implements TopNMetricSpec @Override public byte[] getCacheKey() { - byte[] previousStopBytes = previousStop == null ? new byte[]{} : previousStop.getBytes(Charsets.UTF_8); + byte[] previousStopBytes = previousStop == null ? new byte[]{} : StringUtils.toUtf8(previousStop); return ByteBuffer.allocate(1 + previousStopBytes.length) .put(CACHE_TYPE_ID) diff --git a/processing/src/main/java/io/druid/query/topn/NumericTopNMetricSpec.java b/processing/src/main/java/io/druid/query/topn/NumericTopNMetricSpec.java index ad0a80fe719..cbcc8a01912 100644 --- a/processing/src/main/java/io/druid/query/topn/NumericTopNMetricSpec.java +++ b/processing/src/main/java/io/druid/query/topn/NumericTopNMetricSpec.java @@ -132,7 +132,7 @@ public class NumericTopNMetricSpec implements TopNMetricSpec @Override public byte[] getCacheKey() { - byte[] metricBytes = metric.getBytes(Charsets.UTF_8); + byte[] metricBytes = com.metamx.common.StringUtils.toUtf8(metric); return ByteBuffer.allocate(1 + metricBytes.length) .put(CACHE_TYPE_ID) diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java index 96f1837d075..c51161a7ff0 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -20,6 +20,7 @@ package io.druid.segment.data; import com.google.common.base.Charsets; +import com.google.common.base.Throwables; import com.google.common.collect.Ordering; import com.google.common.primitives.Ints; import com.metamx.common.IAE; @@ -30,6 +31,7 @@ import com.metamx.common.logger.Logger; import java.io.ByteArrayOutputStream; import java.io.Closeable; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.nio.channels.WritableByteChannel; import java.util.Arrays; @@ -377,9 +379,7 @@ public class GenericIndexed implements Indexed, Closeable @Override public String fromByteBuffer(final ByteBuffer buffer, final int numBytes) { - final byte[] bytes = new byte[numBytes]; - buffer.get(bytes); - return new String(bytes, Charsets.UTF_8); + return com.metamx.common.StringUtils.fromUtf8(buffer, numBytes); } @Override @@ -388,7 +388,7 @@ public class GenericIndexed implements Indexed, Closeable if (val == null) { return new byte[]{}; } - return val.getBytes(Charsets.UTF_8); + return com.metamx.common.StringUtils.toUtf8(val); } @Override diff --git a/server/src/main/java/io/druid/client/CacheUtil.java b/server/src/main/java/io/druid/client/CacheUtil.java index 15bd1dc29db..3896d4920d7 100644 --- a/server/src/main/java/io/druid/client/CacheUtil.java +++ b/server/src/main/java/io/druid/client/CacheUtil.java @@ -40,7 +40,7 @@ public class CacheUtil ) { final Interval segmentQueryInterval = descriptor.getInterval(); - final byte[] versionBytes = descriptor.getVersion().getBytes(Charsets.UTF_8); + final byte[] versionBytes = com.metamx.common.StringUtils.toUtf8(descriptor.getVersion()); return new Cache.NamedKey( segmentIdentifier, ByteBuffer diff --git a/server/src/main/java/io/druid/client/ServerInventoryView.java b/server/src/main/java/io/druid/client/ServerInventoryView.java index b95dda08c91..4e7979f1ca0 100644 --- a/server/src/main/java/io/druid/client/ServerInventoryView.java +++ b/server/src/main/java/io/druid/client/ServerInventoryView.java @@ -28,6 +28,7 @@ import com.google.common.base.Throwables; import com.google.common.collect.MapMaker; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; +import com.metamx.common.StringUtils; import com.metamx.emitter.EmittingLogger; import io.druid.concurrent.Execs; import io.druid.curator.inventory.CuratorInventoryManager; @@ -114,6 +115,7 @@ public abstract class ServerInventoryView implements ServerView, return jsonMapper.readValue(bytes, typeReference); } catch (IOException e) { + CharBuffer.wrap(StringUtils.fromUtf8(bytes).toCharArray()); CharBuffer charBuffer = Charsets.UTF_8.decode(ByteBuffer.wrap(bytes)); log.error(e, "Could not parse json: %s", charBuffer.toString()); throw Throwables.propagate(e); diff --git a/server/src/main/java/io/druid/client/cache/Cache.java b/server/src/main/java/io/druid/client/cache/Cache.java index 1ae01596833..e2ddb9592fe 100644 --- a/server/src/main/java/io/druid/client/cache/Cache.java +++ b/server/src/main/java/io/druid/client/cache/Cache.java @@ -22,6 +22,7 @@ package io.druid.client.cache; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; import com.google.common.primitives.Ints; +import com.metamx.common.StringUtils; import java.nio.ByteBuffer; import java.util.Arrays; @@ -54,7 +55,7 @@ public interface Cache } public byte[] toByteArray() { - final byte[] nsBytes = this.namespace.getBytes(Charsets.UTF_8); + final byte[] nsBytes = StringUtils.toUtf8(this.namespace); return ByteBuffer.allocate(Ints.BYTES + nsBytes.length + this.key.length) .putInt(nsBytes.length) .put(nsBytes) diff --git a/server/src/main/java/io/druid/initialization/Initialization.java b/server/src/main/java/io/druid/initialization/Initialization.java index 058a007701d..a5003ab280a 100644 --- a/server/src/main/java/io/druid/initialization/Initialization.java +++ b/server/src/main/java/io/druid/initialization/Initialization.java @@ -32,6 +32,7 @@ import com.google.inject.Module; import com.google.inject.util.Modules; import com.metamx.common.ISE; import com.metamx.common.logger.Logger; +import com.metamx.common.StringUtils; import io.druid.curator.CuratorModule; import io.druid.curator.discovery.DiscoveryModule; import io.druid.guice.AWSModule; @@ -312,7 +313,7 @@ public class Initialization } } - , false, Charsets.UTF_8.name()) + , false, StringUtils.UTF8_STRING) ); return new DefaultTeslaAether( config.getLocalRepository(), diff --git a/server/src/main/java/io/druid/metadata/SQLMetadataStorageActionHandler.java b/server/src/main/java/io/druid/metadata/SQLMetadataStorageActionHandler.java index 2d1fc6b8a7c..818c588a7f3 100644 --- a/server/src/main/java/io/druid/metadata/SQLMetadataStorageActionHandler.java +++ b/server/src/main/java/io/druid/metadata/SQLMetadataStorageActionHandler.java @@ -29,6 +29,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.metamx.common.Pair; import com.metamx.common.RetryUtils; +import com.metamx.common.StringUtils; import com.metamx.emitter.EmittingLogger; import org.joda.time.DateTime; import org.skife.jdbi.v2.FoldController; @@ -398,7 +399,7 @@ public class SQLMetadataStorageActionHandler