From 7afffbe13bca22ff24b1fb307aef4b07a3630de0 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Mon, 10 Jun 2013 16:41:48 +0200 Subject: [PATCH] Cleanup String to UTF-8 conversion Currently we have many different places that convert String to UTF-8 bytes and back. We shouldn't maintain more code than necessary to do this conversion and rather use Lucene's support for it. --- .../search/type/TransportSearchHelper.java | 16 +- .../metadata/MetaDataCreateIndexService.java | 5 +- .../java/org/elasticsearch/common/Names.java | 8 +- .../org/elasticsearch/common/Strings.java | 52 ++-- .../org/elasticsearch/common/Unicode.java | 225 ------------------ .../common/bytes/HashedBytesArray.java | 5 - .../common/compress/CompressedString.java | 5 +- .../common/io/FastByteArrayOutputStream.java | 4 +- .../org/elasticsearch/common/io/Streams.java | 8 +- .../common/lucene/LoggerInfoStream.java | 4 +- .../common/math/MathRuntimeException.java | 4 +- .../common/settings/ImmutableSettings.java | 3 +- .../org/elasticsearch/env/Environment.java | 4 +- .../cache/filter/support/CacheKeyFilter.java | 10 +- .../filter/weighted/WeightedFilterCache.java | 6 +- .../index/cache/id/simple/SimpleIdCache.java | 5 +- .../shard/service/InternalIndexShard.java | 2 +- .../filter/terms/IndicesTermsFilterCache.java | 4 +- .../monitor/dump/AbstractDump.java | 3 +- .../elasticsearch/script/ScriptService.java | 3 +- .../search/builder/SearchSourceBuilder.java | 6 +- .../InternalCountDateHistogramFacet.java | 3 +- .../InternalFullDateHistogramFacet.java | 3 +- .../facet/filter/InternalFilterFacet.java | 3 +- .../geodistance/InternalGeoDistanceFacet.java | 4 +- .../InternalCountHistogramFacet.java | 3 +- .../histogram/InternalFullHistogramFacet.java | 3 +- .../facet/query/InternalQueryFacet.java | 3 +- .../facet/range/InternalRangeFacet.java | 4 +- .../statistical/InternalStatisticalFacet.java | 3 +- .../doubles/InternalDoubleTermsFacet.java | 3 +- .../terms/longs/InternalLongTermsFacet.java | 3 +- .../strings/InternalStringTermsFacet.java | 3 +- .../InternalTermsStatsDoubleFacet.java | 3 +- .../longs/InternalTermsStatsLongFacet.java | 3 +- .../InternalTermsStatsStringFacet.java | 3 +- .../broadcast/BroadcastActionsTests.java | 6 +- .../document/DocumentActionsTests.java | 28 ++- .../integration/rest/helper/HttpClient.java | 6 +- .../basic/TransportSearchFailuresTests.java | 8 +- .../basic/TransportTwoNodesSearchTests.java | 4 +- .../search/suggest/SuggestSearchTests.java | 6 +- .../validate/SimpleValidateQueryTests.java | 8 +- .../unit/action/bulk/BulkRequestTests.java | 10 +- .../test/unit/common/io/StreamsTests.java | 27 ++- .../index/analysis/AnalysisModuleTests.java | 4 +- .../index/cache/id/SimpleIdCacheTests.java | 23 +- .../unit/index/gateway/CommitPointsTests.java | 3 +- .../mapper/simple/SimpleMapperTests.java | 4 +- .../query/SimpleIndexQueryParserTests.java | 3 +- .../phrase/NoisyChannelSpellCheckerTests.java | 8 +- 51 files changed, 216 insertions(+), 361 deletions(-) delete mode 100644 src/main/java/org/elasticsearch/common/Unicode.java diff --git a/src/main/java/org/elasticsearch/action/search/type/TransportSearchHelper.java b/src/main/java/org/elasticsearch/action/search/type/TransportSearchHelper.java index ea7ef97e283..521c3f392ae 100644 --- a/src/main/java/org/elasticsearch/action/search/type/TransportSearchHelper.java +++ b/src/main/java/org/elasticsearch/action/search/type/TransportSearchHelper.java @@ -21,6 +21,10 @@ package org.elasticsearch.action.search.type; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CharsRef; +import org.apache.lucene.util.UnicodeUtil; import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.ElasticSearchIllegalStateException; import org.elasticsearch.action.search.SearchRequest; @@ -30,7 +34,6 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.Base64; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.Unicode; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.internal.InternalScrollSearchRequest; @@ -82,16 +85,21 @@ public abstract class TransportSearchHelper { sb.append(entry.getKey()).append(':').append(entry.getValue()).append(';'); } } - return Base64.encodeBytes(Unicode.fromStringAsBytes(sb.toString()), Base64.URL_SAFE); + BytesRef bytesRef = new BytesRef(); + UnicodeUtil.UTF16toUTF8(sb, 0, sb.length(), bytesRef); + + return Base64.encodeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length, Base64.URL_SAFE); } public static ParsedScrollId parseScrollId(String scrollId) { + CharsRef spare = new CharsRef(); try { - scrollId = Unicode.fromBytes(Base64.decode(scrollId, Base64.URL_SAFE)); + byte[] decode = Base64.decode(scrollId, Base64.URL_SAFE); + UnicodeUtil.UTF8toUTF16(decode, 0, decode.length, spare); } catch (IOException e) { throw new ElasticSearchIllegalArgumentException("Failed to decode scrollId", e); } - String[] elements = Strings.splitStringToArray(scrollId, ';'); + String[] elements = Strings.splitStringToArray(spare, ';'); int index = 0; String type = elements[index++]; int contextSize = Integer.parseInt(elements[index++]); diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 0c0de80a2c5..931a0dee914 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -19,12 +19,12 @@ package org.elasticsearch.cluster.metadata; +import com.google.common.base.Charsets; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.io.Closeables; -import org.apache.lucene.util.UnicodeUtil; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterService; @@ -38,7 +38,6 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.Unicode; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.compress.CompressedString; import org.elasticsearch.common.inject.Inject; @@ -439,7 +438,7 @@ public class MetaDataCreateIndexService extends AbstractComponent { int lastDotIndex = mappingFile.getName().lastIndexOf('.'); String mappingType = lastDotIndex != -1 ? mappingFile.getName().substring(0, lastDotIndex) : mappingFile.getName(); try { - String mappingSource = Streams.copyToString(new InputStreamReader(new FileInputStream(mappingFile), Streams.UTF8)); + String mappingSource = Streams.copyToString(new InputStreamReader(new FileInputStream(mappingFile), Charsets.UTF_8)); if (mappings.containsKey(mappingType)) { XContentHelper.mergeDefaults(mappings.get(mappingType), parseMapping(mappingSource)); } else { diff --git a/src/main/java/org/elasticsearch/common/Names.java b/src/main/java/org/elasticsearch/common/Names.java index 994db851d7f..f5fa2ab6bed 100644 --- a/src/main/java/org/elasticsearch/common/Names.java +++ b/src/main/java/org/elasticsearch/common/Names.java @@ -30,6 +30,8 @@ import java.util.Random; import org.elasticsearch.common.io.Streams; +import com.google.common.base.Charsets; + /** * */ @@ -38,13 +40,13 @@ public abstract class Names { public static String randomNodeName(URL nodeNames) { BufferedReader reader = null; try { - reader = new BufferedReader(new InputStreamReader(nodeNames.openStream(), Streams.UTF8)); + reader = new BufferedReader(new InputStreamReader(nodeNames.openStream(), Charsets.UTF_8)); int numberOfNames = 0; while (reader.readLine() != null) { numberOfNames++; } reader.close(); - reader = new BufferedReader(new InputStreamReader(nodeNames.openStream(), Streams.UTF8)); + reader = new BufferedReader(new InputStreamReader(nodeNames.openStream(), Charsets.UTF_8)); int number = ((ThreadLocalRandom.current().nextInt(numberOfNames)) % numberOfNames); for (int i = 0; i < number; i++) { reader.readLine(); @@ -68,7 +70,7 @@ public abstract class Names { return null; } try { - BufferedReader reader = new BufferedReader(new InputStreamReader(nodeNames, Streams.UTF8)); + BufferedReader reader = new BufferedReader(new InputStreamReader(nodeNames, Charsets.UTF_8)); int numberOfNames = Integer.parseInt(reader.readLine()); int number = ((new Random().nextInt(numberOfNames)) % numberOfNames) - 2; // remove 2 for last line and first line for (int i = 0; i < number; i++) { diff --git a/src/main/java/org/elasticsearch/common/Strings.java b/src/main/java/org/elasticsearch/common/Strings.java index dd903042060..3eb912e7cf4 100644 --- a/src/main/java/org/elasticsearch/common/Strings.java +++ b/src/main/java/org/elasticsearch/common/Strings.java @@ -22,6 +22,9 @@ package org.elasticsearch.common; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import gnu.trove.set.hash.THashSet; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.UnicodeUtil; import org.elasticsearch.common.io.FastStringReader; import java.io.BufferedReader; @@ -1029,39 +1032,37 @@ public class Strings { return result; } - public static String[] splitStringToArray(final String s, final char c) { + public static String[] splitStringToArray(final CharSequence s, final char c) { if (s.length() == 0) { return Strings.EMPTY_ARRAY; } - final char[] chars = s.toCharArray(); int count = 1; - for (final char x : chars) { - if (x == c) { + for (int i = 0; i < s.length(); i++) { + if (s.charAt(i) == c) { count++; } } final String[] result = new String[count]; - final int len = chars.length; - int start = 0; // starting index in chars of the current substring. - int pos = 0; // current index in chars. - int i = 0; // number of the current substring. - for (; pos < len; pos++) { - if (chars[pos] == c) { - int size = pos - start; - if (size > 0) { - result[i++] = new String(chars, start, size); + final StringBuilder builder = new StringBuilder(); + int res = 0; + for (int i = 0; i < s.length(); i++) { + if (s.charAt(i) == c) { + if (builder.length() > 0) { + result[res++] = builder.toString(); + builder.setLength(0); } - start = pos + 1; + + } else { + builder.append(s.charAt(i)); } } - int size = pos - start; - if (size > 0) { - result[i++] = new String(chars, start, size); + if (builder.length() > 0) { + result[res++] = builder.toString(); } - if (i != count) { + if (res != count) { // we have empty strings, copy over to a new array - String[] result1 = new String[i]; - System.arraycopy(result, 0, result1, 0, i); + String[] result1 = new String[res]; + System.arraycopy(result, 0, result1, 0, res); return result1; } return result; @@ -1492,4 +1493,15 @@ public class Strings { private Strings() { } + + public static byte[] toUTF8Bytes(CharSequence charSequence) { + return toUTF8Bytes(charSequence, new BytesRef()); + } + + public static byte[] toUTF8Bytes(CharSequence charSequence, BytesRef spare) { + UnicodeUtil.UTF16toUTF8(charSequence, 0, charSequence.length(), spare); + final byte[] bytes = new byte[spare.length]; + System.arraycopy(spare.bytes, spare.offset, bytes, 0, bytes.length); + return bytes; + } } diff --git a/src/main/java/org/elasticsearch/common/Unicode.java b/src/main/java/org/elasticsearch/common/Unicode.java deleted file mode 100644 index dbe137aba5a..00000000000 --- a/src/main/java/org/elasticsearch/common/Unicode.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common; - -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.UnicodeUtil; - -import java.util.Arrays; - -/** - * - */ -public class Unicode { - - private static ThreadLocal cachedUtf8Result = new ThreadLocal() { - @Override - protected BytesRef initialValue() { - return new BytesRef(); - } - }; - - private static ThreadLocal cachedUtf16Result = new ThreadLocal() { - @Override - protected UTF16Result initialValue() { - return new UTF16Result(); - } - }; - - public static byte[] fromStringAsBytes(String source) { - if (source == null) { - return null; - } - BytesRef result = unsafeFromStringAsUtf8(source); - return Arrays.copyOfRange(result.bytes, result.offset, result.length); - } - - public static BytesRef fromStringAsUtf8(String source) { - if (source == null) { - return null; - } - BytesRef result = new BytesRef(); - UnicodeUtil.UTF16toUTF8(source, 0, source.length(), result); - return result; - } - - public static void fromStringAsUtf8(String source, BytesRef result) { - if (source == null) { - result.length = 0; - return; - } - UnicodeUtil.UTF16toUTF8(source, 0, source.length(), result); - } - - public static BytesRef unsafeFromStringAsUtf8(String source) { - if (source == null) { - return null; - } - BytesRef result = cachedUtf8Result.get(); - UnicodeUtil.UTF16toUTF8(source, 0, source.length(), result); - return result; - } - - public static String fromBytes(byte[] source) { - return fromBytes(source, 0, source.length); - } - - public static String fromBytes(byte[] source, int offset, int length) { - if (source == null) { - return null; - } - UTF16Result result = unsafeFromBytesAsUtf16(source, offset, length); - return new String(result.result, 0, result.length); - } - - public static UTF16Result fromBytesAsUtf16(byte[] source) { - return fromBytesAsUtf16(source, 0, source.length); - } - - public static UTF16Result fromBytesAsUtf16(byte[] source, int offset, int length) { - if (source == null) { - return null; - } - UTF16Result result = new UTF16Result(); - UTF8toUTF16(source, offset, length, result); - return result; - } - - public static UTF16Result unsafeFromBytesAsUtf16(byte[] source) { - return unsafeFromBytesAsUtf16(source, 0, source.length); - } - - public static UTF16Result unsafeFromBytesAsUtf16(byte[] source, int offset, int length) { - if (source == null) { - return null; - } - UTF16Result result = cachedUtf16Result.get(); - UTF8toUTF16(source, offset, length, result); - return result; - } - - // LUCENE MONITOR - - // an optimized version of UTF16Result that does not hold the offsets since we don't need them - // they are only used with continuous writing to the same utf16 (without "clearing it") - - public static final class UTF16Result { - public char[] result = new char[10]; - // public int[] offsets = new int[10]; - public int length; - - public void setLength(int newLength) { - if (result.length < newLength) { - char[] newArray = new char[(int) (1.5 * newLength)]; - System.arraycopy(result, 0, newArray, 0, length); - result = newArray; - } - length = newLength; - } - - public void copyText(UTF16Result other) { - setLength(other.length); - System.arraycopy(other.result, 0, result, 0, length); - } - } - - - /** - * Convert UTF8 bytes into UTF16 characters. If offset - * is non-zero, conversion starts at that starting point - * in utf8, re-using the results from the previous call - * up until offset. - */ - public static void UTF8toUTF16(final byte[] utf8, final int offset, final int length, final UTF16Result result) { - - final int end = offset + length; - char[] out = result.result; -// if (result.offsets.length <= end) { -// int[] newOffsets = new int[2 * end]; -// System.arraycopy(result.offsets, 0, newOffsets, 0, result.offsets.length); -// result.offsets = newOffsets; -// } -// final int[] offsets = result.offsets; - - // If incremental decoding fell in the middle of a - // single unicode character, rollback to its start: - int upto = offset; -// while (offsets[upto] == -1) -// upto--; - - int outUpto = 0; // offsets[upto]; - - // Pre-allocate for worst case 1-for-1 - if (outUpto + length >= out.length) { - char[] newOut = new char[2 * (outUpto + length)]; - System.arraycopy(out, 0, newOut, 0, outUpto); - result.result = out = newOut; - } - - while (upto < end) { - - final int b = utf8[upto] & 0xff; - final int ch; - - upto += 1; // CHANGE -// offsets[upto++] = outUpto; - - if (b < 0xc0) { - assert b < 0x80; - ch = b; - } else if (b < 0xe0) { - ch = ((b & 0x1f) << 6) + (utf8[upto] & 0x3f); - upto += 1; // CHANGE -// offsets[upto++] = -1; - } else if (b < 0xf0) { - ch = ((b & 0xf) << 12) + ((utf8[upto] & 0x3f) << 6) + (utf8[upto + 1] & 0x3f); - upto += 2; // CHANGE -// offsets[upto++] = -1; -// offsets[upto++] = -1; - } else { - assert b < 0xf8; - ch = ((b & 0x7) << 18) + ((utf8[upto] & 0x3f) << 12) + ((utf8[upto + 1] & 0x3f) << 6) + (utf8[upto + 2] & 0x3f); - upto += 3; // CHANGE -// offsets[upto++] = -1; -// offsets[upto++] = -1; -// offsets[upto++] = -1; - } - - if (ch <= UNI_MAX_BMP) { - // target is a character <= 0xFFFF - out[outUpto++] = (char) ch; - } else { - // target is a character in range 0xFFFF - 0x10FFFF - final int chHalf = ch - HALF_BASE; - out[outUpto++] = (char) ((chHalf >> HALF_SHIFT) + UnicodeUtil.UNI_SUR_HIGH_START); - out[outUpto++] = (char) ((chHalf & HALF_MASK) + UnicodeUtil.UNI_SUR_LOW_START); - } - } - -// offsets[upto] = outUpto; - result.length = outUpto; - } - - private static final long UNI_MAX_BMP = 0x0000FFFF; - - private static final int HALF_BASE = 0x0010000; - private static final long HALF_SHIFT = 10; - private static final long HALF_MASK = 0x3FFL; -} diff --git a/src/main/java/org/elasticsearch/common/bytes/HashedBytesArray.java b/src/main/java/org/elasticsearch/common/bytes/HashedBytesArray.java index b0d45cd242c..7f3bec5b467 100644 --- a/src/main/java/org/elasticsearch/common/bytes/HashedBytesArray.java +++ b/src/main/java/org/elasticsearch/common/bytes/HashedBytesArray.java @@ -22,7 +22,6 @@ package org.elasticsearch.common.bytes; import com.google.common.base.Charsets; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticSearchIllegalArgumentException; -import org.elasticsearch.common.Unicode; import org.elasticsearch.common.io.stream.BytesStreamInput; import org.elasticsearch.common.io.stream.StreamInput; import org.jboss.netty.buffer.ChannelBuffer; @@ -47,10 +46,6 @@ public class HashedBytesArray implements BytesReference { this.bytes = bytes; } - public HashedBytesArray(String str) { - this(Unicode.fromStringAsBytes(str)); - } - @Override public byte get(int index) { return bytes[index]; diff --git a/src/main/java/org/elasticsearch/common/compress/CompressedString.java b/src/main/java/org/elasticsearch/common/compress/CompressedString.java index 54427b4f191..cc836ef596c 100644 --- a/src/main/java/org/elasticsearch/common/compress/CompressedString.java +++ b/src/main/java/org/elasticsearch/common/compress/CompressedString.java @@ -20,7 +20,6 @@ package org.elasticsearch.common.compress; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.Unicode; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; @@ -79,7 +78,7 @@ public class CompressedString implements Streamable { } public CompressedString(String str) throws IOException { - BytesRef result = Unicode.unsafeFromStringAsUtf8(str); + BytesRef result = new BytesRef(str); this.bytes = CompressorFactory.defaultCompressor().compress(result.bytes, result.offset, result.length); } @@ -93,7 +92,7 @@ public class CompressedString implements Streamable { } public String string() throws IOException { - return Unicode.fromBytes(uncompressed()); + return new BytesRef(uncompressed()).utf8ToString(); } public static CompressedString readCompressedString(StreamInput in) throws IOException { diff --git a/src/main/java/org/elasticsearch/common/io/FastByteArrayOutputStream.java b/src/main/java/org/elasticsearch/common/io/FastByteArrayOutputStream.java index c140d9ea7de..1951b156ac6 100644 --- a/src/main/java/org/elasticsearch/common/io/FastByteArrayOutputStream.java +++ b/src/main/java/org/elasticsearch/common/io/FastByteArrayOutputStream.java @@ -27,6 +27,8 @@ import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import com.google.common.base.Charsets; + /** * Similar to {@link java.io.ByteArrayOutputStream} just not synced. */ @@ -171,7 +173,7 @@ public class FastByteArrayOutputStream extends OutputStream implements BytesStre * @since JDK1.1 */ public String toString() { - return new String(buf, 0, count, Streams.UTF8); + return new String(buf, 0, count, Charsets.UTF_8); } /** diff --git a/src/main/java/org/elasticsearch/common/io/Streams.java b/src/main/java/org/elasticsearch/common/io/Streams.java index 6593dbac971..f6098ef723e 100644 --- a/src/main/java/org/elasticsearch/common/io/Streams.java +++ b/src/main/java/org/elasticsearch/common/io/Streams.java @@ -23,6 +23,8 @@ import org.elasticsearch.common.Preconditions; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.CachedStreamOutput; +import com.google.common.base.Charsets; + import java.io.*; import java.nio.charset.Charset; @@ -36,8 +38,6 @@ import java.nio.charset.Charset; */ public abstract class Streams { - public static final Charset UTF8 = Charset.forName("UTF-8"); - public static final int BUFFER_SIZE = 1024 * 8; @@ -255,7 +255,7 @@ public abstract class Streams { if (is == null) { throw new FileNotFoundException("Resource [" + path + "] not found in classpath with class loader [" + classLoader + "]"); } - return copyToString(new InputStreamReader(is, UTF8)); + return copyToString(new InputStreamReader(is, Charsets.UTF_8)); } public static String copyToStringFromClasspath(String path) throws IOException { @@ -263,7 +263,7 @@ public abstract class Streams { if (is == null) { throw new FileNotFoundException("Resource [" + path + "] not found in classpath"); } - return copyToString(new InputStreamReader(is, UTF8)); + return copyToString(new InputStreamReader(is, Charsets.UTF_8)); } public static byte[] copyToBytesFromClasspath(String path) throws IOException { diff --git a/src/main/java/org/elasticsearch/common/lucene/LoggerInfoStream.java b/src/main/java/org/elasticsearch/common/lucene/LoggerInfoStream.java index f8eba8cb008..f7845260d95 100644 --- a/src/main/java/org/elasticsearch/common/lucene/LoggerInfoStream.java +++ b/src/main/java/org/elasticsearch/common/lucene/LoggerInfoStream.java @@ -23,6 +23,8 @@ import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; +import com.google.common.base.Charsets; + import java.io.OutputStream; import java.io.PrintStream; import java.io.UnsupportedEncodingException; @@ -74,7 +76,7 @@ public class LoggerInfoStream extends PrintStream { * @throws UnsupportedEncodingException */ public LoggerInfoStream(ESLogger logger) throws UnsupportedEncodingException { - super((OutputStream) null, false, Streams.UTF8.name()); + super((OutputStream) null, false, Charsets.UTF_8.name()); this.logger = logger; } diff --git a/src/main/java/org/elasticsearch/common/math/MathRuntimeException.java b/src/main/java/org/elasticsearch/common/math/MathRuntimeException.java index 87de0168b78..53db567160f 100644 --- a/src/main/java/org/elasticsearch/common/math/MathRuntimeException.java +++ b/src/main/java/org/elasticsearch/common/math/MathRuntimeException.java @@ -31,6 +31,8 @@ import java.util.*; import org.elasticsearch.common.io.Streams; +import com.google.common.base.Charsets; + /** * Base class for commons-math unchecked exceptions. * @@ -197,7 +199,7 @@ public class MathRuntimeException extends RuntimeException { @Override public void printStackTrace(final PrintStream out) { synchronized (out) { - PrintWriter pw = new PrintWriter(new OutputStreamWriter(out, Streams.UTF8)); + PrintWriter pw = new PrintWriter(new OutputStreamWriter(out, Charsets.UTF_8)); printStackTrace(pw); // Flush the PrintWriter before it's GC'ed. pw.flush(); diff --git a/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java b/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java index 6c020c4fca0..a685bdd7273 100644 --- a/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java +++ b/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.settings; +import com.google.common.base.Charsets; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -747,7 +748,7 @@ public class ImmutableSettings implements Settings { public Builder loadFromStream(String resourceName, InputStream is) throws SettingsException { SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromResource(resourceName); try { - Map loadedSettings = settingsLoader.load(Streams.copyToString(new InputStreamReader(is, Streams.UTF8))); + Map loadedSettings = settingsLoader.load(Streams.copyToString(new InputStreamReader(is, Charsets.UTF_8))); put(loadedSettings); } catch (Exception e) { throw new SettingsException("Failed to load settings from [" + resourceName + "]", e); diff --git a/src/main/java/org/elasticsearch/env/Environment.java b/src/main/java/org/elasticsearch/env/Environment.java index d4742f3f9a6..a364329d049 100644 --- a/src/main/java/org/elasticsearch/env/Environment.java +++ b/src/main/java/org/elasticsearch/env/Environment.java @@ -23,6 +23,8 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; +import com.google.common.base.Charsets; + import java.io.File; import java.io.IOException; import java.io.InputStreamReader; @@ -164,7 +166,7 @@ public class Environment { } public String resolveConfigAndLoadToString(String path) throws FailedToResolveConfigException, IOException { - return Streams.copyToString(new InputStreamReader(resolveConfig(path).openStream(), Streams.UTF8)); + return Streams.copyToString(new InputStreamReader(resolveConfig(path).openStream(), Charsets.UTF_8)); } public URL resolveConfig(String path) throws FailedToResolveConfigException { diff --git a/src/main/java/org/elasticsearch/index/cache/filter/support/CacheKeyFilter.java b/src/main/java/org/elasticsearch/index/cache/filter/support/CacheKeyFilter.java index 646348ff7f8..70fd272d0e1 100644 --- a/src/main/java/org/elasticsearch/index/cache/filter/support/CacheKeyFilter.java +++ b/src/main/java/org/elasticsearch/index/cache/filter/support/CacheKeyFilter.java @@ -23,7 +23,7 @@ import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Filter; import org.apache.lucene.util.Bits; -import org.elasticsearch.common.Unicode; +import org.elasticsearch.common.Strings; import java.io.IOException; import java.util.Arrays; @@ -43,17 +43,13 @@ public interface CacheKeyFilter { } public Key(String str) { - this(Unicode.fromStringAsBytes(str)); + this(Strings.toUTF8Bytes(str)); } public byte[] bytes() { return this.bytes; } - - public String utf8ToString() { - return Unicode.fromBytes(bytes); - } - + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/src/main/java/org/elasticsearch/index/cache/filter/weighted/WeightedFilterCache.java b/src/main/java/org/elasticsearch/index/cache/filter/weighted/WeightedFilterCache.java index 6eced5df453..d02d76a7019 100644 --- a/src/main/java/org/elasticsearch/index/cache/filter/weighted/WeightedFilterCache.java +++ b/src/main/java/org/elasticsearch/index/cache/filter/weighted/WeightedFilterCache.java @@ -28,8 +28,10 @@ import org.apache.lucene.index.SegmentReader; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Filter; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.docset.DocIdSets; import org.elasticsearch.common.lucene.search.CachedFilter; @@ -93,9 +95,11 @@ public class WeightedFilterCache extends AbstractIndexComponent implements Filte @Override public void clear(String reason, String[] keys) { logger.debug("clear keys [], reason [{}]", reason, keys); + final BytesRef spare = new BytesRef(); for (String key : keys) { + final byte[] keyBytes = Strings.toUTF8Bytes(key, spare); for (Object readerKey : seenReaders.keySet()) { - indicesFilterCache.cache().invalidate(new FilterCacheKey(readerKey, new CacheKeyFilter.Key(key))); + indicesFilterCache.cache().invalidate(new FilterCacheKey(readerKey, new CacheKeyFilter.Key(keyBytes))); } } } diff --git a/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java b/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java index 2ea582219ad..5eaa0da0959 100644 --- a/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java +++ b/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java @@ -22,7 +22,9 @@ package org.elasticsearch.index.cache.id.simple; import gnu.trove.impl.Constants; import org.apache.lucene.index.*; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.UnicodeUtil; import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.HashedBytesArray; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.inject.Inject; @@ -119,10 +121,11 @@ public class SimpleIdCache extends AbstractIndexComponent implements IdCache, Se // We don't want to load uid of child documents, this allows us to not load uids of child types. NavigableSet parentTypes = new TreeSet(UTF8SortedAsUnicodeComparator.utf8SortedAsUnicodeSortOrder); + BytesRef spare = new BytesRef(); for (String type : indexService.mapperService().types()) { ParentFieldMapper parentFieldMapper = indexService.mapperService().documentMapper(type).parentFieldMapper(); if (parentFieldMapper != null) { - parentTypes.add(new HashedBytesArray(parentFieldMapper.type())); + parentTypes.add(new HashedBytesArray(Strings.toUTF8Bytes(parentFieldMapper.type(), spare))); } } diff --git a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java index 3d9056c5331..aa438b4cb6c 100644 --- a/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java +++ b/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java @@ -841,7 +841,7 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I } CheckIndex checkIndex = new CheckIndex(store.directory()); FastByteArrayOutputStream os = new FastByteArrayOutputStream(); - PrintStream out = new PrintStream(os, false, Streams.UTF8.name()); + PrintStream out = new PrintStream(os, false, Charsets.UTF_8.name()); checkIndex.setInfoStream(out); out.flush(); CheckIndex.Status status = checkIndex.checkIndex(); diff --git a/src/main/java/org/elasticsearch/indices/cache/filter/terms/IndicesTermsFilterCache.java b/src/main/java/org/elasticsearch/indices/cache/filter/terms/IndicesTermsFilterCache.java index 765af8178a1..bd472b7cd91 100644 --- a/src/main/java/org/elasticsearch/indices/cache/filter/terms/IndicesTermsFilterCache.java +++ b/src/main/java/org/elasticsearch/indices/cache/filter/terms/IndicesTermsFilterCache.java @@ -32,6 +32,7 @@ import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -133,8 +134,9 @@ public class IndicesTermsFilterCache extends AbstractComponent { } public void clear(String reason, String[] keys) { + final BytesRef spare = new BytesRef(); for (String key : keys) { - cache.invalidate(new CacheKeyFilter.Key(key)); + cache.invalidate(new CacheKeyFilter.Key(Strings.toUTF8Bytes(key, spare))); } } diff --git a/src/main/java/org/elasticsearch/monitor/dump/AbstractDump.java b/src/main/java/org/elasticsearch/monitor/dump/AbstractDump.java index f14cc175f73..206cdc05bf3 100644 --- a/src/main/java/org/elasticsearch/monitor/dump/AbstractDump.java +++ b/src/main/java/org/elasticsearch/monitor/dump/AbstractDump.java @@ -19,6 +19,7 @@ package org.elasticsearch.monitor.dump; +import com.google.common.base.Charsets; import com.google.common.collect.ImmutableMap; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.Streams; @@ -89,6 +90,6 @@ public abstract class AbstractDump implements Dump { @Override public Writer createFileWriter(String name) throws DumpException { - return new OutputStreamWriter(createFileOutputStream(name), Streams.UTF8); + return new OutputStreamWriter(createFileOutputStream(name), Charsets.UTF_8); } } diff --git a/src/main/java/org/elasticsearch/script/ScriptService.java b/src/main/java/org/elasticsearch/script/ScriptService.java index 53f073f3fce..6869d6a532e 100644 --- a/src/main/java/org/elasticsearch/script/ScriptService.java +++ b/src/main/java/org/elasticsearch/script/ScriptService.java @@ -19,6 +19,7 @@ package org.elasticsearch.script; +import com.google.common.base.Charsets; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.collect.ImmutableMap; @@ -121,7 +122,7 @@ public class ScriptService extends AbstractComponent { if (s.equals(ext)) { found = true; try { - String script = Streams.copyToString(new InputStreamReader(new FileInputStream(file), Streams.UTF8)); + String script = Streams.copyToString(new InputStreamReader(new FileInputStream(file), Charsets.UTF_8)); staticCache.put(scriptName, new CompiledScript(engineService.types()[0], engineService.compile(script))); } catch (Exception e) { logger.warn("failed to load/compile script [{}]", e, scriptName); diff --git a/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index a980182a1b2..52758a93865 100644 --- a/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.builder; +import com.google.common.base.Charsets; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import gnu.trove.iterator.TObjectFloatIterator; @@ -26,7 +27,6 @@ import gnu.trove.map.hash.TObjectFloatHashMap; import org.elasticsearch.ElasticSearchGenerationException; import org.elasticsearch.client.Requests; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.Unicode; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.TimeValue; @@ -156,7 +156,7 @@ public class SearchSourceBuilder implements ToXContent { * Constructs a new search source builder with a raw search query. */ public SearchSourceBuilder query(String queryString) { - return query(Unicode.fromStringAsBytes(queryString)); + return query(queryString.getBytes(Charsets.UTF_8)); } /** @@ -193,7 +193,7 @@ public class SearchSourceBuilder implements ToXContent { * (and not facets for example). */ public SearchSourceBuilder filter(String filterString) { - return filter(Unicode.fromStringAsBytes(filterString)); + return filter(filterString.getBytes(Charsets.UTF_8)); } /** diff --git a/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalCountDateHistogramFacet.java b/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalCountDateHistogramFacet.java index f131ae1444a..4e23b38412f 100644 --- a/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalCountDateHistogramFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalCountDateHistogramFacet.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.facet.datehistogram; import gnu.trove.iterator.TLongLongIterator; import gnu.trove.map.hash.TLongLongHashMap; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.HashedBytesArray; import org.elasticsearch.common.io.stream.StreamInput; @@ -40,7 +41,7 @@ import java.util.List; */ public class InternalCountDateHistogramFacet extends InternalDateHistogramFacet { - private static final BytesReference STREAM_TYPE = new HashedBytesArray("cdHistogram"); + private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("cdHistogram")); public static void registerStreams() { Streams.registerStream(STREAM, STREAM_TYPE); diff --git a/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalFullDateHistogramFacet.java b/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalFullDateHistogramFacet.java index 5d61d961935..7842b09ad1a 100644 --- a/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalFullDateHistogramFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalFullDateHistogramFacet.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.facet.datehistogram; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.HashedBytesArray; import org.elasticsearch.common.io.stream.StreamInput; @@ -37,7 +38,7 @@ import java.util.*; */ public class InternalFullDateHistogramFacet extends InternalDateHistogramFacet { - private static final BytesReference STREAM_TYPE = new HashedBytesArray("fdHistogram"); + private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("fdHistogram")); public static void registerStreams() { Streams.registerStream(STREAM, STREAM_TYPE); diff --git a/src/main/java/org/elasticsearch/search/facet/filter/InternalFilterFacet.java b/src/main/java/org/elasticsearch/search/facet/filter/InternalFilterFacet.java index 956e89ba230..1697874d211 100644 --- a/src/main/java/org/elasticsearch/search/facet/filter/InternalFilterFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/filter/InternalFilterFacet.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.facet.filter; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.HashedBytesArray; import org.elasticsearch.common.io.stream.StreamInput; @@ -36,7 +37,7 @@ import java.util.List; */ public class InternalFilterFacet extends InternalFacet implements FilterFacet { - private static final BytesReference STREAM_TYPE = new HashedBytesArray("filter"); + private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("filter")); public static void registerStreams() { Streams.registerStream(STREAM, STREAM_TYPE); diff --git a/src/main/java/org/elasticsearch/search/facet/geodistance/InternalGeoDistanceFacet.java b/src/main/java/org/elasticsearch/search/facet/geodistance/InternalGeoDistanceFacet.java index d98986f7a3a..3885377ac0e 100644 --- a/src/main/java/org/elasticsearch/search/facet/geodistance/InternalGeoDistanceFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/geodistance/InternalGeoDistanceFacet.java @@ -20,6 +20,8 @@ package org.elasticsearch.search.facet.geodistance; import com.google.common.collect.ImmutableList; + +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.HashedBytesArray; import org.elasticsearch.common.io.stream.StreamInput; @@ -38,7 +40,7 @@ import java.util.List; */ public class InternalGeoDistanceFacet extends InternalFacet implements GeoDistanceFacet { - private static final BytesReference STREAM_TYPE = new HashedBytesArray("geoDistance"); + private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("geoDistance")); public static void registerStreams() { Streams.registerStream(STREAM, STREAM_TYPE); diff --git a/src/main/java/org/elasticsearch/search/facet/histogram/InternalCountHistogramFacet.java b/src/main/java/org/elasticsearch/search/facet/histogram/InternalCountHistogramFacet.java index d36327fc97d..5e46a29d6e8 100644 --- a/src/main/java/org/elasticsearch/search/facet/histogram/InternalCountHistogramFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/histogram/InternalCountHistogramFacet.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.facet.histogram; import gnu.trove.iterator.TLongLongIterator; import gnu.trove.map.hash.TLongLongHashMap; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.HashedBytesArray; import org.elasticsearch.common.io.stream.StreamInput; @@ -40,7 +41,7 @@ import java.util.List; */ public class InternalCountHistogramFacet extends InternalHistogramFacet { - private static final BytesReference STREAM_TYPE = new HashedBytesArray("cHistogram"); + private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("cHistogram")); public static void registerStreams() { Streams.registerStream(STREAM, STREAM_TYPE); diff --git a/src/main/java/org/elasticsearch/search/facet/histogram/InternalFullHistogramFacet.java b/src/main/java/org/elasticsearch/search/facet/histogram/InternalFullHistogramFacet.java index ac5960bb915..0e322e1ff9f 100644 --- a/src/main/java/org/elasticsearch/search/facet/histogram/InternalFullHistogramFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/histogram/InternalFullHistogramFacet.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.facet.histogram; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.HashedBytesArray; import org.elasticsearch.common.io.stream.StreamInput; @@ -37,7 +38,7 @@ import java.util.*; */ public class InternalFullHistogramFacet extends InternalHistogramFacet { - private static final BytesReference STREAM_TYPE = new HashedBytesArray("fHistogram"); + private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("fHistogram")); public static void registerStreams() { Streams.registerStream(STREAM, STREAM_TYPE); diff --git a/src/main/java/org/elasticsearch/search/facet/query/InternalQueryFacet.java b/src/main/java/org/elasticsearch/search/facet/query/InternalQueryFacet.java index 5987fe1ad9b..e87edd6cdce 100644 --- a/src/main/java/org/elasticsearch/search/facet/query/InternalQueryFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/query/InternalQueryFacet.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.facet.query; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.HashedBytesArray; import org.elasticsearch.common.io.stream.StreamInput; @@ -36,7 +37,7 @@ import java.util.List; */ public class InternalQueryFacet extends InternalFacet implements QueryFacet { - private static final BytesReference STREAM_TYPE = new HashedBytesArray("query"); + private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("query")); public static void registerStreams() { Streams.registerStream(STREAM, STREAM_TYPE); diff --git a/src/main/java/org/elasticsearch/search/facet/range/InternalRangeFacet.java b/src/main/java/org/elasticsearch/search/facet/range/InternalRangeFacet.java index 9b9e85b27af..8667c6e5a3b 100644 --- a/src/main/java/org/elasticsearch/search/facet/range/InternalRangeFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/range/InternalRangeFacet.java @@ -20,6 +20,8 @@ package org.elasticsearch.search.facet.range; import com.google.common.collect.ImmutableList; + +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.HashedBytesArray; import org.elasticsearch.common.io.stream.StreamInput; @@ -38,7 +40,7 @@ import java.util.List; */ public class InternalRangeFacet extends InternalFacet implements RangeFacet { - private static final BytesReference STREAM_TYPE = new HashedBytesArray("range"); + private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("range")); public static void registerStreams() { Streams.registerStream(STREAM, STREAM_TYPE); diff --git a/src/main/java/org/elasticsearch/search/facet/statistical/InternalStatisticalFacet.java b/src/main/java/org/elasticsearch/search/facet/statistical/InternalStatisticalFacet.java index 119a3a975e5..540cdef05bc 100644 --- a/src/main/java/org/elasticsearch/search/facet/statistical/InternalStatisticalFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/statistical/InternalStatisticalFacet.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.facet.statistical; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.HashedBytesArray; import org.elasticsearch.common.io.stream.StreamInput; @@ -36,7 +37,7 @@ import java.util.List; */ public class InternalStatisticalFacet extends InternalFacet implements StatisticalFacet { - private static final BytesReference STREAM_TYPE = new HashedBytesArray("statistical"); + private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("statistical")); public static void registerStreams() { Streams.registerStream(STREAM, STREAM_TYPE); diff --git a/src/main/java/org/elasticsearch/search/facet/terms/doubles/InternalDoubleTermsFacet.java b/src/main/java/org/elasticsearch/search/facet/terms/doubles/InternalDoubleTermsFacet.java index 328fcd8192e..eb6fba5d67f 100644 --- a/src/main/java/org/elasticsearch/search/facet/terms/doubles/InternalDoubleTermsFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/terms/doubles/InternalDoubleTermsFacet.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList; import gnu.trove.iterator.TDoubleIntIterator; import gnu.trove.map.hash.TDoubleIntHashMap; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.HashedBytesArray; import org.elasticsearch.common.collect.BoundedTreeSet; @@ -47,7 +48,7 @@ import java.util.List; */ public class InternalDoubleTermsFacet extends InternalTermsFacet { - private static final BytesReference STREAM_TYPE = new HashedBytesArray("dTerms"); + private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("dTerms")); public static void registerStream() { Streams.registerStream(STREAM, STREAM_TYPE); diff --git a/src/main/java/org/elasticsearch/search/facet/terms/longs/InternalLongTermsFacet.java b/src/main/java/org/elasticsearch/search/facet/terms/longs/InternalLongTermsFacet.java index 7e0e650e2cd..408733318dc 100644 --- a/src/main/java/org/elasticsearch/search/facet/terms/longs/InternalLongTermsFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/terms/longs/InternalLongTermsFacet.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList; import gnu.trove.iterator.TLongIntIterator; import gnu.trove.map.hash.TLongIntHashMap; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.HashedBytesArray; import org.elasticsearch.common.collect.BoundedTreeSet; @@ -47,7 +48,7 @@ import java.util.List; */ public class InternalLongTermsFacet extends InternalTermsFacet { - private static final BytesReference STREAM_TYPE = new HashedBytesArray("lTerms"); + private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("lTerms")); public static void registerStream() { Streams.registerStream(STREAM, STREAM_TYPE); diff --git a/src/main/java/org/elasticsearch/search/facet/terms/strings/InternalStringTermsFacet.java b/src/main/java/org/elasticsearch/search/facet/terms/strings/InternalStringTermsFacet.java index 44542d81594..e98fa98fcf1 100644 --- a/src/main/java/org/elasticsearch/search/facet/terms/strings/InternalStringTermsFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/terms/strings/InternalStringTermsFacet.java @@ -24,6 +24,7 @@ import gnu.trove.iterator.TObjectIntIterator; import gnu.trove.map.hash.TObjectIntHashMap; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.HashedBytesArray; @@ -50,7 +51,7 @@ import java.util.List; */ public class InternalStringTermsFacet extends InternalTermsFacet { - private static final BytesReference STREAM_TYPE = new HashedBytesArray("tTerms"); + private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("tTerms")); public static void registerStream() { Streams.registerStream(STREAM, STREAM_TYPE); diff --git a/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/InternalTermsStatsDoubleFacet.java b/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/InternalTermsStatsDoubleFacet.java index a42e96abd85..1c6e9c735cc 100644 --- a/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/InternalTermsStatsDoubleFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/InternalTermsStatsDoubleFacet.java @@ -21,6 +21,7 @@ package org.elasticsearch.search.facet.termsstats.doubles; import com.google.common.collect.ImmutableList; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.HashedBytesArray; import org.elasticsearch.common.io.stream.StreamInput; @@ -38,7 +39,7 @@ import java.util.*; public class InternalTermsStatsDoubleFacet extends InternalTermsStatsFacet { - private static final BytesReference STREAM_TYPE = new HashedBytesArray("dTS"); + private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("dTS")); public static void registerStream() { Streams.registerStream(STREAM, STREAM_TYPE); diff --git a/src/main/java/org/elasticsearch/search/facet/termsstats/longs/InternalTermsStatsLongFacet.java b/src/main/java/org/elasticsearch/search/facet/termsstats/longs/InternalTermsStatsLongFacet.java index 6d5a9544caf..d4f6ca53468 100644 --- a/src/main/java/org/elasticsearch/search/facet/termsstats/longs/InternalTermsStatsLongFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/termsstats/longs/InternalTermsStatsLongFacet.java @@ -21,6 +21,7 @@ package org.elasticsearch.search.facet.termsstats.longs; import com.google.common.collect.ImmutableList; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.HashedBytesArray; import org.elasticsearch.common.io.stream.StreamInput; @@ -38,7 +39,7 @@ import java.util.*; public class InternalTermsStatsLongFacet extends InternalTermsStatsFacet { - private static final BytesReference STREAM_TYPE = new HashedBytesArray("lTS"); + private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("lTS")); public static void registerStream() { Streams.registerStream(STREAM, STREAM_TYPE); diff --git a/src/main/java/org/elasticsearch/search/facet/termsstats/strings/InternalTermsStatsStringFacet.java b/src/main/java/org/elasticsearch/search/facet/termsstats/strings/InternalTermsStatsStringFacet.java index 5b70afc3d67..7682685e9b0 100644 --- a/src/main/java/org/elasticsearch/search/facet/termsstats/strings/InternalTermsStatsStringFacet.java +++ b/src/main/java/org/elasticsearch/search/facet/termsstats/strings/InternalTermsStatsStringFacet.java @@ -21,6 +21,7 @@ package org.elasticsearch.search.facet.termsstats.strings; import com.google.common.collect.ImmutableList; import org.elasticsearch.common.CacheRecycler; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.HashedBytesArray; @@ -40,7 +41,7 @@ import java.util.*; public class InternalTermsStatsStringFacet extends InternalTermsStatsFacet { - private static final BytesReference STREAM_TYPE = new HashedBytesArray("tTS"); + private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("tTS")); public static void registerStream() { Streams.registerStream(STREAM, STREAM_TYPE); diff --git a/src/test/java/org/elasticsearch/test/integration/broadcast/BroadcastActionsTests.java b/src/test/java/org/elasticsearch/test/integration/broadcast/BroadcastActionsTests.java index 588d4b54637..3f96594a7ba 100644 --- a/src/test/java/org/elasticsearch/test/integration/broadcast/BroadcastActionsTests.java +++ b/src/test/java/org/elasticsearch/test/integration/broadcast/BroadcastActionsTests.java @@ -26,12 +26,14 @@ import org.elasticsearch.action.admin.indices.flush.FlushResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; -import org.elasticsearch.common.Unicode; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.test.integration.AbstractSharedClusterTest; import org.testng.annotations.Test; +import com.beust.jcommander.Strings; +import com.google.common.base.Charsets; + import java.io.IOException; import static org.elasticsearch.client.Requests.*; @@ -95,7 +97,7 @@ public class BroadcastActionsTests extends AbstractSharedClusterTest { for (int i = 0; i < 5; i++) { // test failed (simply query that can't be parsed) - CountResponse countResponse = client().count(countRequest("test").query(Unicode.fromStringAsBytes("{ term : { _type : \"type1 } }"))).actionGet(); + CountResponse countResponse = client().count(countRequest("test").query("{ term : { _type : \"type1 } }".getBytes(Charsets.UTF_8))).actionGet(); assertThat(countResponse.getCount(), equalTo(0l)); assertThat(countResponse.getTotalShards(), equalTo(5)); diff --git a/src/test/java/org/elasticsearch/test/integration/document/DocumentActionsTests.java b/src/test/java/org/elasticsearch/test/integration/document/DocumentActionsTests.java index 8f9d26e99d1..94b2be4af96 100644 --- a/src/test/java/org/elasticsearch/test/integration/document/DocumentActionsTests.java +++ b/src/test/java/org/elasticsearch/test/integration/document/DocumentActionsTests.java @@ -19,6 +19,21 @@ package org.elasticsearch.test.integration.document; +import static org.elasticsearch.client.Requests.clearIndicesCacheRequest; +import static org.elasticsearch.client.Requests.clusterHealthRequest; +import static org.elasticsearch.client.Requests.countRequest; +import static org.elasticsearch.client.Requests.getRequest; +import static org.elasticsearch.client.Requests.indexRequest; +import static org.elasticsearch.client.Requests.refreshRequest; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; + +import java.io.IOException; +import java.util.Map; + import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; @@ -33,22 +48,13 @@ import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading; import org.elasticsearch.action.support.replication.ReplicationType; -import org.elasticsearch.common.Unicode; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.test.integration.AbstractSharedClusterTest; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.io.IOException; -import java.util.Map; - -import static org.elasticsearch.client.Requests.*; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.nullValue; +import com.google.common.base.Charsets; /** * @@ -196,7 +202,7 @@ public class DocumentActionsTests extends AbstractSharedClusterTest { assertThat(countResponse.getFailedShards(), equalTo(0)); // test failed (simply query that can't be parsed) - countResponse = client().count(countRequest("test").query(Unicode.fromStringAsBytes("{ term : { _type : \"type1 } }"))).actionGet(); + countResponse = client().count(countRequest("test").query("{ term : { _type : \"type1 } }".getBytes(Charsets.UTF_8))).actionGet(); assertThat(countResponse.getCount(), equalTo(0l)); assertThat(countResponse.getSuccessfulShards(), equalTo(0)); diff --git a/src/test/java/org/elasticsearch/test/integration/rest/helper/HttpClient.java b/src/test/java/org/elasticsearch/test/integration/rest/helper/HttpClient.java index 600f6e7ac87..816ae704ab4 100644 --- a/src/test/java/org/elasticsearch/test/integration/rest/helper/HttpClient.java +++ b/src/test/java/org/elasticsearch/test/integration/rest/helper/HttpClient.java @@ -23,6 +23,8 @@ import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.transport.TransportAddress; +import com.google.common.base.Charsets; + import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -96,7 +98,7 @@ public class HttpClient { InputStream inputStream = urlConnection.getInputStream(); String body = null; try { - body = Streams.copyToString(new InputStreamReader(inputStream, Streams.UTF8)); + body = Streams.copyToString(new InputStreamReader(inputStream, Charsets.UTF_8)); } catch (IOException e1) { throw new ElasticSearchException("problem reading error stream", e1); } @@ -105,7 +107,7 @@ public class HttpClient { InputStream errStream = urlConnection.getErrorStream(); String body = null; try { - body = Streams.copyToString(new InputStreamReader(errStream, Streams.UTF8)); + body = Streams.copyToString(new InputStreamReader(errStream, Charsets.UTF_8)); } catch (IOException e1) { throw new ElasticSearchException("problem reading error stream", e1); } diff --git a/src/test/java/org/elasticsearch/test/integration/search/basic/TransportSearchFailuresTests.java b/src/test/java/org/elasticsearch/test/integration/search/basic/TransportSearchFailuresTests.java index c6b91713f65..40555ad3709 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/basic/TransportSearchFailuresTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/basic/TransportSearchFailuresTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.test.integration.search.basic; import static org.elasticsearch.client.Requests.clusterHealthRequest; -import static org.elasticsearch.client.Requests.createIndexRequest; import static org.elasticsearch.client.Requests.refreshRequest; import static org.elasticsearch.client.Requests.searchRequest; import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; @@ -41,11 +40,12 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; import org.elasticsearch.common.Priority; -import org.elasticsearch.common.Unicode; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.integration.AbstractSharedClusterTest; import org.testng.annotations.Test; +import com.google.common.base.Charsets; + /** * */ @@ -69,7 +69,7 @@ public class TransportSearchFailuresTests extends AbstractSharedClusterTest { assertThat(refreshResponse.getFailedShards(), equalTo(0)); for (int i = 0; i < 5; i++) { try { - SearchResponse searchResponse = client().search(searchRequest("test").source(Unicode.fromStringAsBytes("{ xxx }"))).actionGet(); + SearchResponse searchResponse = client().search(searchRequest("test").source("{ xxx }".getBytes(Charsets.UTF_8))).actionGet(); assertThat(searchResponse.getTotalShards(), equalTo(3)); assertThat(searchResponse.getSuccessfulShards(), equalTo(0)); assertThat(searchResponse.getFailedShards(), equalTo(3)); @@ -98,7 +98,7 @@ public class TransportSearchFailuresTests extends AbstractSharedClusterTest { for (int i = 0; i < 5; i++) { try { - SearchResponse searchResponse = client().search(searchRequest("test").source(Unicode.fromStringAsBytes("{ xxx }"))).actionGet(); + SearchResponse searchResponse = client().search(searchRequest("test").source("{ xxx }".getBytes(Charsets.UTF_8))).actionGet(); assertThat(searchResponse.getTotalShards(), equalTo(3)); assertThat(searchResponse.getSuccessfulShards(), equalTo(0)); assertThat(searchResponse.getFailedShards(), equalTo(3)); diff --git a/src/test/java/org/elasticsearch/test/integration/search/basic/TransportTwoNodesSearchTests.java b/src/test/java/org/elasticsearch/test/integration/search/basic/TransportTwoNodesSearchTests.java index 90b7eba2652..d50acea5e46 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/basic/TransportTwoNodesSearchTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/basic/TransportTwoNodesSearchTests.java @@ -50,7 +50,6 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.Requests; import org.elasticsearch.common.Priority; -import org.elasticsearch.common.Unicode; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.Scroll; @@ -64,6 +63,7 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import com.google.common.base.Charsets; import com.google.common.collect.Sets; /** @@ -339,7 +339,7 @@ public class TransportTwoNodesSearchTests extends AbstractSharedClusterTest { public void testFailedSearchWithWrongQuery() throws Exception { logger.info("Start Testing failed search with wrong query"); try { - SearchResponse searchResponse = client().search(searchRequest("test").source(Unicode.fromStringAsBytes("{ xxx }"))).actionGet(); + SearchResponse searchResponse = client().search(searchRequest("test").source("{ xxx }".getBytes(Charsets.UTF_8))).actionGet(); assertThat(searchResponse.getTotalShards(), equalTo(3)); assertThat(searchResponse.getSuccessfulShards(), equalTo(0)); assertThat(searchResponse.getFailedShards(), equalTo(3)); diff --git a/src/test/java/org/elasticsearch/test/integration/search/suggest/SuggestSearchTests.java b/src/test/java/org/elasticsearch/test/integration/search/suggest/SuggestSearchTests.java index 7e6b024388d..cd5641ac761 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/suggest/SuggestSearchTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/suggest/SuggestSearchTests.java @@ -56,6 +56,8 @@ import org.elasticsearch.test.integration.AbstractSharedClusterTest; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; +import com.google.common.base.Charsets; + /** */ public class SuggestSearchTests extends AbstractSharedClusterTest { @@ -550,7 +552,7 @@ public class SuggestSearchTests extends AbstractSharedClusterTest { client().admin().indices().prepareCreate("test").setSettings(builder.build()).addMapping("type1", mapping).execute().actionGet(); client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet(); - BufferedReader reader = new BufferedReader(new InputStreamReader(SuggestSearchTests.class.getResourceAsStream("/config/names.txt"), Streams.UTF8)); + BufferedReader reader = new BufferedReader(new InputStreamReader(SuggestSearchTests.class.getResourceAsStream("/config/names.txt"), Charsets.UTF_8)); String line = null; while ((line = reader.readLine()) != null) { client().prepareIndex("test", "type1") @@ -884,7 +886,7 @@ public class SuggestSearchTests extends AbstractSharedClusterTest { client().admin().indices().prepareCreate("test").setSettings(builder.build()).addMapping("type1", mapping).execute().actionGet(); client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet(); - BufferedReader reader = new BufferedReader(new InputStreamReader(SuggestSearchTests.class.getResourceAsStream("/config/names.txt"), Streams.UTF8)); + BufferedReader reader = new BufferedReader(new InputStreamReader(SuggestSearchTests.class.getResourceAsStream("/config/names.txt"), Charsets.UTF_8)); String line = null; while ((line = reader.readLine()) != null) { client().prepareIndex("test", "type1") diff --git a/src/test/java/org/elasticsearch/test/integration/validate/SimpleValidateQueryTests.java b/src/test/java/org/elasticsearch/test/integration/validate/SimpleValidateQueryTests.java index 509c4a2043e..9592ba40dcd 100644 --- a/src/test/java/org/elasticsearch/test/integration/validate/SimpleValidateQueryTests.java +++ b/src/test/java/org/elasticsearch/test/integration/validate/SimpleValidateQueryTests.java @@ -35,6 +35,8 @@ import org.hamcrest.Matcher; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; +import com.google.common.base.Charsets; + import java.io.IOException; import static org.hamcrest.MatcherAssert.assertThat; @@ -60,7 +62,7 @@ public class SimpleValidateQueryTests extends AbstractSharedClusterTest { client().admin().indices().prepareRefresh().execute().actionGet(); - assertThat(client().admin().indices().prepareValidateQuery("test").setQuery("foo".getBytes(Streams.UTF8)).execute().actionGet().isValid(), equalTo(false)); + assertThat(client().admin().indices().prepareValidateQuery("test").setQuery("foo".getBytes(Charsets.UTF_8)).execute().actionGet().isValid(), equalTo(false)); assertThat(client().admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryString("_id:1")).execute().actionGet().isValid(), equalTo(true)); assertThat(client().admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryString("_i:d:1")).execute().actionGet().isValid(), equalTo(false)); @@ -100,7 +102,7 @@ public class SimpleValidateQueryTests extends AbstractSharedClusterTest { ValidateQueryResponse response; response = client().admin().indices().prepareValidateQuery("test") - .setQuery("foo".getBytes(Streams.UTF8)) + .setQuery("foo".getBytes(Charsets.UTF_8)) .setExplain(true) .execute().actionGet(); assertThat(response.isValid(), equalTo(false)); @@ -207,7 +209,7 @@ public class SimpleValidateQueryTests extends AbstractSharedClusterTest { for (Client client : cluster().clients()) { ValidateQueryResponse response = client.admin().indices().prepareValidateQuery("test") - .setQuery("foo".getBytes(Streams.UTF8)) + .setQuery("foo".getBytes(Charsets.UTF_8)) .setExplain(true) .execute().actionGet(); assertThat(response.isValid(), equalTo(false)); diff --git a/src/test/java/org/elasticsearch/test/unit/action/bulk/BulkRequestTests.java b/src/test/java/org/elasticsearch/test/unit/action/bulk/BulkRequestTests.java index 3f60116ae33..cda4339a142 100644 --- a/src/test/java/org/elasticsearch/test/unit/action/bulk/BulkRequestTests.java +++ b/src/test/java/org/elasticsearch/test/unit/action/bulk/BulkRequestTests.java @@ -27,6 +27,8 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.Streams; import org.testng.annotations.Test; +import com.google.common.base.Charsets; + import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; @@ -38,7 +40,7 @@ public class BulkRequestTests { public void testSimpleBulk1() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/test/unit/action/bulk/simple-bulk.json"); BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(bulkAction.getBytes(Streams.UTF8), 0, bulkAction.length(), true, null, null); + bulkRequest.add(bulkAction.getBytes(Charsets.UTF_8), 0, bulkAction.length(), true, null, null); assertThat(bulkRequest.numberOfActions(), equalTo(3)); assertThat(((IndexRequest) bulkRequest.requests().get(0)).source().toBytes(), equalTo(new BytesArray("{ \"field1\" : \"value1\" }").toBytes())); assertThat(bulkRequest.requests().get(1), instanceOf(DeleteRequest.class)); @@ -49,7 +51,7 @@ public class BulkRequestTests { public void testSimpleBulk2() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/test/unit/action/bulk/simple-bulk2.json"); BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(bulkAction.getBytes(Streams.UTF8), 0, bulkAction.length(), true, null, null); + bulkRequest.add(bulkAction.getBytes(Charsets.UTF_8), 0, bulkAction.length(), true, null, null); assertThat(bulkRequest.numberOfActions(), equalTo(3)); } @@ -57,7 +59,7 @@ public class BulkRequestTests { public void testSimpleBulk3() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/test/unit/action/bulk/simple-bulk3.json"); BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(bulkAction.getBytes(Streams.UTF8), 0, bulkAction.length(), true, null, null); + bulkRequest.add(bulkAction.getBytes(Charsets.UTF_8), 0, bulkAction.length(), true, null, null); assertThat(bulkRequest.numberOfActions(), equalTo(3)); } @@ -65,7 +67,7 @@ public class BulkRequestTests { public void testSimpleBulk4() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/test/unit/action/bulk/simple-bulk4.json"); BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(bulkAction.getBytes(Streams.UTF8), 0, bulkAction.length(), true, null, null); + bulkRequest.add(bulkAction.getBytes(Charsets.UTF_8), 0, bulkAction.length(), true, null, null); assertThat(bulkRequest.numberOfActions(), equalTo(4)); assertThat(((UpdateRequest) bulkRequest.requests().get(0)).id(), equalTo("1")); assertThat(((UpdateRequest) bulkRequest.requests().get(0)).retryOnConflict(), equalTo(2)); diff --git a/src/test/java/org/elasticsearch/test/unit/common/io/StreamsTests.java b/src/test/java/org/elasticsearch/test/unit/common/io/StreamsTests.java index de375e34bd8..8f9b05a2f76 100644 --- a/src/test/java/org/elasticsearch/test/unit/common/io/StreamsTests.java +++ b/src/test/java/org/elasticsearch/test/unit/common/io/StreamsTests.java @@ -19,16 +19,23 @@ package org.elasticsearch.test.unit.common.io; -import org.elasticsearch.common.io.Streams; -import org.testng.annotations.Test; - -import java.io.*; -import java.util.Arrays; - -import static org.elasticsearch.common.io.Streams.*; +import static org.elasticsearch.common.io.Streams.copy; +import static org.elasticsearch.common.io.Streams.copyToByteArray; +import static org.elasticsearch.common.io.Streams.copyToString; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.Arrays; + +import org.testng.annotations.Test; + +import com.google.common.base.Charsets; + /** * Unit tests for {@link org.elasticsearch.common.io.Streams}. * @@ -38,7 +45,7 @@ public class StreamsTests { @Test public void testCopyFromInputStream() throws IOException { - byte[] content = "content".getBytes(Streams.UTF8); + byte[] content = "content".getBytes(Charsets.UTF_8); ByteArrayInputStream in = new ByteArrayInputStream(content); ByteArrayOutputStream out = new ByteArrayOutputStream(content.length); long count = copy(in, out); @@ -49,7 +56,7 @@ public class StreamsTests { @Test public void testCopyFromByteArray() throws IOException { - byte[] content = "content".getBytes(Streams.UTF8); + byte[] content = "content".getBytes(Charsets.UTF_8); ByteArrayOutputStream out = new ByteArrayOutputStream(content.length); copy(content, out); assertThat(Arrays.equals(content, out.toByteArray()), equalTo(true)); @@ -57,7 +64,7 @@ public class StreamsTests { @Test public void testCopyToByteArray() throws IOException { - byte[] content = "content".getBytes(Streams.UTF8); + byte[] content = "content".getBytes(Charsets.UTF_8); ByteArrayInputStream in = new ByteArrayInputStream(content); byte[] result = copyToByteArray(in); assertThat(Arrays.equals(content, result), equalTo(true)); diff --git a/src/test/java/org/elasticsearch/test/unit/index/analysis/AnalysisModuleTests.java b/src/test/java/org/elasticsearch/test/unit/index/analysis/AnalysisModuleTests.java index e985ce95d30..0389ba7ec79 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/analysis/AnalysisModuleTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/analysis/AnalysisModuleTests.java @@ -45,6 +45,8 @@ import org.elasticsearch.test.unit.index.analysis.filter1.MyFilterTokenFilterFac import org.hamcrest.MatcherAssert; import org.testng.annotations.Test; +import com.google.common.base.Charsets; + import java.io.BufferedWriter; import java.io.File; import java.io.FileOutputStream; @@ -172,7 +174,7 @@ public class AnalysisModuleTests { BufferedWriter writer = null; try { - writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(wordListFile), Streams.UTF8)); + writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(wordListFile), Charsets.UTF_8)); for (String word : words) { writer.write(word); writer.write('\n'); diff --git a/src/test/java/org/elasticsearch/test/unit/index/cache/id/SimpleIdCacheTests.java b/src/test/java/org/elasticsearch/test/unit/index/cache/id/SimpleIdCacheTests.java index b3ce03e98ba..207869bb870 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/cache/id/SimpleIdCacheTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/cache/id/SimpleIdCacheTests.java @@ -30,6 +30,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.store.RAMDirectory; import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.HashedBytesArray; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Injector; @@ -132,10 +133,10 @@ public class SimpleIdCacheTests { assertThat(typeCache.parentIdByDoc(3).toUtf8(), equalTo("1")); assertThat(typeCache.parentIdByDoc(4).toUtf8(), equalTo("1")); - assertThat(typeCache.docById(new HashedBytesArray("1")), equalTo(0)); - assertThat(typeCache.docById(new HashedBytesArray("2")), equalTo(-1)); - assertThat(typeCache.docById(new HashedBytesArray("3")), equalTo(-1)); - assertThat(typeCache.docById(new HashedBytesArray("4")), equalTo(-1)); + assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("1"))), equalTo(0)); + assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("2"))), equalTo(-1)); + assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("3"))), equalTo(-1)); + assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("4"))), equalTo(-1)); // Verify simple id cache for segment 2 readerCache = idCache.reader(leaves.get(1).reader()); @@ -153,10 +154,10 @@ public class SimpleIdCacheTests { assertThat(typeCache.parentIdByDoc(3), nullValue()); assertThat(typeCache.parentIdByDoc(4), nullValue()); - assertThat(typeCache.docById(new HashedBytesArray("2")), equalTo(1)); - assertThat(typeCache.docById(new HashedBytesArray("3")), equalTo(2)); - assertThat(typeCache.docById(new HashedBytesArray("4")), equalTo(3)); - assertThat(typeCache.docById(new HashedBytesArray("5")), equalTo(4)); + assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("2"))), equalTo(1)); + assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("3"))), equalTo(2)); + assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("4"))), equalTo(3)); + assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("5"))), equalTo(4)); // Verify simple id cache for segment 3 readerCache = idCache.reader(leaves.get(2).reader()); @@ -176,8 +177,8 @@ public class SimpleIdCacheTests { assertThat(typeCache.parentIdByDoc(4).toUtf8(), equalTo("4")); assertThat(typeCache.parentIdByDoc(5), nullValue()); - assertThat(typeCache.docById(new HashedBytesArray("6")), equalTo(0)); - assertThat(typeCache.docById(new HashedBytesArray("7")), equalTo(5)); + assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("6"))), equalTo(0)); + assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("7"))), equalTo(5)); // Verify simple id cache for segment 4 readerCache = idCache.reader(leaves.get(3).reader()); @@ -195,7 +196,7 @@ public class SimpleIdCacheTests { assertThat(typeCache.parentIdByDoc(3), nullValue()); assertThat(typeCache.parentIdByDoc(4), nullValue()); - assertThat(typeCache.docById(new HashedBytesArray("8")), equalTo(4)); + assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("8"))), equalTo(4)); } @Test(expectedExceptions = AssertionError.class) diff --git a/src/test/java/org/elasticsearch/test/unit/index/gateway/CommitPointsTests.java b/src/test/java/org/elasticsearch/test/unit/index/gateway/CommitPointsTests.java index 4c7594d9bd7..88a205b0a25 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/gateway/CommitPointsTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/gateway/CommitPointsTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.test.unit.index.gateway; +import com.google.common.base.Charsets; import com.google.common.collect.Lists; import org.elasticsearch.common.io.Streams; @@ -54,7 +55,7 @@ public class CommitPointsTests { CommitPoint commitPoint = new CommitPoint(1, "test", CommitPoint.Type.GENERATED, indexFiles, translogFiles); byte[] serialized = CommitPoints.toXContent(commitPoint); - logger.info("serialized commit_point {}", new String(serialized, Streams.UTF8)); + logger.info("serialized commit_point {}", new String(serialized, Charsets.UTF_8)); CommitPoint desCp = CommitPoints.fromXContent(serialized); assertThat(desCp.version(), equalTo(commitPoint.version())); diff --git a/src/test/java/org/elasticsearch/test/unit/index/mapper/simple/SimpleMapperTests.java b/src/test/java/org/elasticsearch/test/unit/index/mapper/simple/SimpleMapperTests.java index 569748e2997..d6efed50eff 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/mapper/simple/SimpleMapperTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/mapper/simple/SimpleMapperTests.java @@ -30,6 +30,8 @@ import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.test.unit.index.mapper.MapperTests; import org.testng.annotations.Test; +import com.google.common.base.Charsets; + import static org.elasticsearch.common.io.Streams.copyToBytesFromClasspath; import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath; import static org.elasticsearch.index.mapper.MapperBuilders.*; @@ -124,7 +126,7 @@ public class SimpleMapperTests { .add(object("name").add(stringField("first").store(true).index(false))) ).build(mapperParser); - BytesReference json = new BytesArray("".getBytes(Streams.UTF8)); + BytesReference json = new BytesArray("".getBytes(Charsets.UTF_8)); try { docMapper.parse("person", "1", json).rootDoc(); assertThat("this point is never reached", false); diff --git a/src/test/java/org/elasticsearch/test/unit/index/query/SimpleIndexQueryParserTests.java b/src/test/java/org/elasticsearch/test/unit/index/query/SimpleIndexQueryParserTests.java index 651b31b65eb..bc2761ed9bf 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/query/SimpleIndexQueryParserTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/query/SimpleIndexQueryParserTests.java @@ -31,6 +31,7 @@ import org.apache.lucene.spatial.prefix.IntersectsPrefixTreeFilter; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.Injector; @@ -759,7 +760,7 @@ public class SimpleIndexQueryParserTests { Filter filter = ((XFilteredQuery) parsedQuery.query()).getFilter(); assertThat(filter, instanceOf(CacheKeyFilter.Wrapper.class)); CacheKeyFilter.Wrapper wrapper = (CacheKeyFilter.Wrapper) filter; - assertThat(wrapper.cacheKey().utf8ToString(), equalTo("key")); + assertThat(new BytesRef(wrapper.cacheKey().bytes()).utf8ToString(), equalTo("key")); assertThat(wrapper.wrappedFilter(), instanceOf(RegexpFilter.class)); RegexpFilter regexpFilter = (RegexpFilter) wrapper.wrappedFilter(); assertThat(regexpFilter.field(), equalTo("name.first")); diff --git a/src/test/java/org/elasticsearch/test/unit/search/suggest/phrase/NoisyChannelSpellCheckerTests.java b/src/test/java/org/elasticsearch/test/unit/search/suggest/phrase/NoisyChannelSpellCheckerTests.java index 5c9805a8a4c..85eebd73579 100644 --- a/src/test/java/org/elasticsearch/test/unit/search/suggest/phrase/NoisyChannelSpellCheckerTests.java +++ b/src/test/java/org/elasticsearch/test/unit/search/suggest/phrase/NoisyChannelSpellCheckerTests.java @@ -61,6 +61,8 @@ import org.elasticsearch.search.suggest.phrase.NoisyChannelSpellChecker; import org.elasticsearch.search.suggest.phrase.StupidBackoffScorer; import org.elasticsearch.search.suggest.phrase.WordScorer; import org.testng.annotations.Test; + +import com.google.common.base.Charsets; public class NoisyChannelSpellCheckerTests { @Test @@ -94,7 +96,7 @@ public class NoisyChannelSpellCheckerTests { IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_41, wrapper); IndexWriter writer = new IndexWriter(dir, conf); - BufferedReader reader = new BufferedReader(new InputStreamReader(NoisyChannelSpellCheckerTests.class.getResourceAsStream("/config/names.txt"), Streams.UTF8)); + BufferedReader reader = new BufferedReader(new InputStreamReader(NoisyChannelSpellCheckerTests.class.getResourceAsStream("/config/names.txt"), Charsets.UTF_8)); String line = null; while ((line = reader.readLine()) != null) { Document doc = new Document(); @@ -205,7 +207,7 @@ public class NoisyChannelSpellCheckerTests { IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_41, wrapper); IndexWriter writer = new IndexWriter(dir, conf); - BufferedReader reader = new BufferedReader(new InputStreamReader(NoisyChannelSpellCheckerTests.class.getResourceAsStream("/config/names.txt"), Streams.UTF8)); + BufferedReader reader = new BufferedReader(new InputStreamReader(NoisyChannelSpellCheckerTests.class.getResourceAsStream("/config/names.txt"), Charsets.UTF_8)); String line = null; while ((line = reader.readLine()) != null) { Document doc = new Document(); @@ -290,7 +292,7 @@ public class NoisyChannelSpellCheckerTests { IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_41, wrapper); IndexWriter writer = new IndexWriter(dir, conf); - BufferedReader reader = new BufferedReader(new InputStreamReader(NoisyChannelSpellCheckerTests.class.getResourceAsStream("/config/names.txt"), Streams.UTF8)); + BufferedReader reader = new BufferedReader(new InputStreamReader(NoisyChannelSpellCheckerTests.class.getResourceAsStream("/config/names.txt"), Charsets.UTF_8)); String line = null; while ((line = reader.readLine()) != null) { Document doc = new Document();