honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cleanup String to UTF-8 conversion 

Currently we have many different places that convert String to UTF-8
bytes and back. We shouldn't maintain more code than necessary to
do this conversion and rather use Lucene's support for it.
This commit is contained in:
Simon Willnauer 2013-06-10 16:41:48 +02:00
parent 9323e677bd
commit 7afffbe13b
51 changed files with 216 additions and 361 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/main/java/org/elasticsearch/action/search/type/TransportSearchHelper.java
View File

@ -21,6 +21,10 @@ package org.elasticsearch.action.search.type;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.UnicodeUtil;
import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.ElasticSearchIllegalStateException;
import org.elasticsearch.action.search.SearchRequest;
@ -30,7 +34,6 @@ import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.Base64;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.Unicode;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.search.SearchPhaseResult;
import org.elasticsearch.search.internal.InternalScrollSearchRequest;
@ -82,16 +85,21 @@ public abstract class TransportSearchHelper {
sb.append(entry.getKey()).append(':').append(entry.getValue()).append(';');
}
}
return Base64.encodeBytes(Unicode.fromStringAsBytes(sb.toString()), Base64.URL_SAFE);
BytesRef bytesRef = new BytesRef();
UnicodeUtil.UTF16toUTF8(sb, 0, sb.length(), bytesRef);
return Base64.encodeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length, Base64.URL_SAFE);
}
public static ParsedScrollId parseScrollId(String scrollId) {
CharsRef spare = new CharsRef();
try {
scrollId = Unicode.fromBytes(Base64.decode(scrollId, Base64.URL_SAFE));
byte[] decode = Base64.decode(scrollId, Base64.URL_SAFE);
UnicodeUtil.UTF8toUTF16(decode, 0, decode.length, spare);
} catch (IOException e) {
throw new ElasticSearchIllegalArgumentException("Failed to decode scrollId", e);
}
String[] elements = Strings.splitStringToArray(scrollId, ';');
String[] elements = Strings.splitStringToArray(spare, ';');
int index = 0;
String type = elements[index++];
int contextSize = Integer.parseInt(elements[index++]);

5
src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View File

@ -19,12 +19,12 @@
package org.elasticsearch.cluster.metadata;
import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.io.Closeables;
import org.apache.lucene.util.UnicodeUtil;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterService;
@ -38,7 +38,6 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService;
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.Unicode;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.inject.Inject;
@ -439,7 +438,7 @@ public class MetaDataCreateIndexService extends AbstractComponent {
int lastDotIndex = mappingFile.getName().lastIndexOf('.');
String mappingType = lastDotIndex != -1 ? mappingFile.getName().substring(0, lastDotIndex) : mappingFile.getName();
try {
String mappingSource = Streams.copyToString(new InputStreamReader(new FileInputStream(mappingFile), Streams.UTF8));
String mappingSource = Streams.copyToString(new InputStreamReader(new FileInputStream(mappingFile), Charsets.UTF_8));
if (mappings.containsKey(mappingType)) {
XContentHelper.mergeDefaults(mappings.get(mappingType), parseMapping(mappingSource));
} else {

8
src/main/java/org/elasticsearch/common/Names.java
View File

@ -30,6 +30,8 @@ import java.util.Random;
import org.elasticsearch.common.io.Streams;
import com.google.common.base.Charsets;
/**
*
*/
@ -38,13 +40,13 @@ public abstract class Names {
public static String randomNodeName(URL nodeNames) {
BufferedReader reader = null;
try {
reader = new BufferedReader(new InputStreamReader(nodeNames.openStream(), Streams.UTF8));
reader = new BufferedReader(new InputStreamReader(nodeNames.openStream(), Charsets.UTF_8));
int numberOfNames = 0;
while (reader.readLine() != null) {
numberOfNames++;
}
reader.close();
reader = new BufferedReader(new InputStreamReader(nodeNames.openStream(), Streams.UTF8));
reader = new BufferedReader(new InputStreamReader(nodeNames.openStream(), Charsets.UTF_8));
int number = ((ThreadLocalRandom.current().nextInt(numberOfNames)) % numberOfNames);
for (int i = 0; i < number; i++) {
reader.readLine();
@ -68,7 +70,7 @@ public abstract class Names {
return null;
}
try {
BufferedReader reader = new BufferedReader(new InputStreamReader(nodeNames, Streams.UTF8));
BufferedReader reader = new BufferedReader(new InputStreamReader(nodeNames, Charsets.UTF_8));
int numberOfNames = Integer.parseInt(reader.readLine());
int number = ((new Random().nextInt(numberOfNames)) % numberOfNames) - 2; // remove 2 for last line and first line
for (int i = 0; i < number; i++) {

52
src/main/java/org/elasticsearch/common/Strings.java
View File

@ -22,6 +22,9 @@ package org.elasticsearch.common;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import gnu.trove.set.hash.THashSet;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.UnicodeUtil;
import org.elasticsearch.common.io.FastStringReader;
import java.io.BufferedReader;
@ -1029,39 +1032,37 @@ public class Strings {
return result;
}
public static String[] splitStringToArray(final String s, final char c) {
public static String[] splitStringToArray(final CharSequence s, final char c) {
if (s.length() == 0) {
return Strings.EMPTY_ARRAY;
}
final char[] chars = s.toCharArray();
int count = 1;
for (final char x : chars) {
if (x == c) {
for (int i = 0; i < s.length(); i++) {
if (s.charAt(i) == c) {
count++;
}
}
final String[] result = new String[count];
final int len = chars.length;
int start = 0; // starting index in chars of the current substring.
int pos = 0; // current index in chars.
int i = 0; // number of the current substring.
for (; pos < len; pos++) {
if (chars[pos] == c) {
int size = pos - start;
if (size > 0) {
result[i++] = new String(chars, start, size);
final StringBuilder builder = new StringBuilder();
int res = 0;
for (int i = 0; i < s.length(); i++) {
if (s.charAt(i) == c) {
if (builder.length() > 0) {
result[res++] = builder.toString();
builder.setLength(0);
}
start = pos + 1;
} else {
builder.append(s.charAt(i));
}
}
int size = pos - start;
if (size > 0) {
result[i++] = new String(chars, start, size);
if (builder.length() > 0) {
result[res++] = builder.toString();
}
if (i != count) {
if (res != count) {
// we have empty strings, copy over to a new array
String[] result1 = new String[i];
System.arraycopy(result, 0, result1, 0, i);
String[] result1 = new String[res];
System.arraycopy(result, 0, result1, 0, res);
return result1;
}
return result;
@ -1492,4 +1493,15 @@ public class Strings {
private Strings() {
}
public static byte[] toUTF8Bytes(CharSequence charSequence) {
return toUTF8Bytes(charSequence, new BytesRef());
}
public static byte[] toUTF8Bytes(CharSequence charSequence, BytesRef spare) {
UnicodeUtil.UTF16toUTF8(charSequence, 0, charSequence.length(), spare);
final byte[] bytes = new byte[spare.length];
System.arraycopy(spare.bytes, spare.offset, bytes, 0, bytes.length);
return bytes;
}
}

225
src/main/java/org/elasticsearch/common/Unicode.java
View File

@ -1,225 +0,0 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.UnicodeUtil;
import java.util.Arrays;
/**
*
*/
public class Unicode {
private static ThreadLocal<BytesRef> cachedUtf8Result = new ThreadLocal<BytesRef>() {
@Override
protected BytesRef initialValue() {
return new BytesRef();
}
};
private static ThreadLocal<UTF16Result> cachedUtf16Result = new ThreadLocal<UTF16Result>() {
@Override
protected UTF16Result initialValue() {
return new UTF16Result();
}
};
public static byte[] fromStringAsBytes(String source) {
if (source == null) {
return null;
}
BytesRef result = unsafeFromStringAsUtf8(source);
return Arrays.copyOfRange(result.bytes, result.offset, result.length);
}
public static BytesRef fromStringAsUtf8(String source) {
if (source == null) {
return null;
}
BytesRef result = new BytesRef();
UnicodeUtil.UTF16toUTF8(source, 0, source.length(), result);
return result;
}
public static void fromStringAsUtf8(String source, BytesRef result) {
if (source == null) {
result.length = 0;
return;
}
UnicodeUtil.UTF16toUTF8(source, 0, source.length(), result);
}
public static BytesRef unsafeFromStringAsUtf8(String source) {
if (source == null) {
return null;
}
BytesRef result = cachedUtf8Result.get();
UnicodeUtil.UTF16toUTF8(source, 0, source.length(), result);
return result;
}
public static String fromBytes(byte[] source) {
return fromBytes(source, 0, source.length);
}
public static String fromBytes(byte[] source, int offset, int length) {
if (source == null) {
return null;
}
UTF16Result result = unsafeFromBytesAsUtf16(source, offset, length);
return new String(result.result, 0, result.length);
}
public static UTF16Result fromBytesAsUtf16(byte[] source) {
return fromBytesAsUtf16(source, 0, source.length);
}
public static UTF16Result fromBytesAsUtf16(byte[] source, int offset, int length) {
if (source == null) {
return null;
}
UTF16Result result = new UTF16Result();
UTF8toUTF16(source, offset, length, result);
return result;
}
public static UTF16Result unsafeFromBytesAsUtf16(byte[] source) {
return unsafeFromBytesAsUtf16(source, 0, source.length);
}
public static UTF16Result unsafeFromBytesAsUtf16(byte[] source, int offset, int length) {
if (source == null) {
return null;
}
UTF16Result result = cachedUtf16Result.get();
UTF8toUTF16(source, offset, length, result);
return result;
}
// LUCENE MONITOR
// an optimized version of UTF16Result that does not hold the offsets since we don't need them
// they are only used with continuous writing to the same utf16 (without "clearing it")
public static final class UTF16Result {
public char[] result = new char[10];
// public int[] offsets = new int[10];
public int length;
public void setLength(int newLength) {
if (result.length < newLength) {
char[] newArray = new char[(int) (1.5 * newLength)];
System.arraycopy(result, 0, newArray, 0, length);
result = newArray;
}
length = newLength;
}
public void copyText(UTF16Result other) {
setLength(other.length);
System.arraycopy(other.result, 0, result, 0, length);
}
}
/**
* Convert UTF8 bytes into UTF16 characters. If offset
* is non-zero, conversion starts at that starting point
* in utf8, re-using the results from the previous call
* up until offset.
*/
public static void UTF8toUTF16(final byte[] utf8, final int offset, final int length, final UTF16Result result) {
final int end = offset + length;
char[] out = result.result;
// if (result.offsets.length <= end) {
// int[] newOffsets = new int[2 * end];
// System.arraycopy(result.offsets, 0, newOffsets, 0, result.offsets.length);
// result.offsets = newOffsets;
// }
// final int[] offsets = result.offsets;
// If incremental decoding fell in the middle of a
// single unicode character, rollback to its start:
int upto = offset;
// while (offsets[upto] == -1)
// upto--;
int outUpto = 0; // offsets[upto];
// Pre-allocate for worst case 1-for-1
if (outUpto + length >= out.length) {
char[] newOut = new char[2 * (outUpto + length)];
System.arraycopy(out, 0, newOut, 0, outUpto);
result.result = out = newOut;
}
while (upto < end) {
final int b = utf8[upto] & 0xff;
final int ch;
upto += 1; // CHANGE
// offsets[upto++] = outUpto;
if (b < 0xc0) {
assert b < 0x80;
ch = b;
} else if (b < 0xe0) {
ch = ((b & 0x1f) << 6) + (utf8[upto] & 0x3f);
upto += 1; // CHANGE
// offsets[upto++] = -1;
} else if (b < 0xf0) {
ch = ((b & 0xf) << 12) + ((utf8[upto] & 0x3f) << 6) + (utf8[upto + 1] & 0x3f);
upto += 2; // CHANGE
// offsets[upto++] = -1;
// offsets[upto++] = -1;
} else {
assert b < 0xf8;
ch = ((b & 0x7) << 18) + ((utf8[upto] & 0x3f) << 12) + ((utf8[upto + 1] & 0x3f) << 6) + (utf8[upto + 2] & 0x3f);
upto += 3; // CHANGE
// offsets[upto++] = -1;
// offsets[upto++] = -1;
// offsets[upto++] = -1;
}
if (ch <= UNI_MAX_BMP) {
// target is a character <= 0xFFFF
out[outUpto++] = (char) ch;
} else {
// target is a character in range 0xFFFF - 0x10FFFF
final int chHalf = ch - HALF_BASE;
out[outUpto++] = (char) ((chHalf >> HALF_SHIFT) + UnicodeUtil.UNI_SUR_HIGH_START);
out[outUpto++] = (char) ((chHalf & HALF_MASK) + UnicodeUtil.UNI_SUR_LOW_START);
}
}
// offsets[upto] = outUpto;
result.length = outUpto;
}
private static final long UNI_MAX_BMP = 0x0000FFFF;
private static final int HALF_BASE = 0x0010000;
private static final long HALF_SHIFT = 10;
private static final long HALF_MASK = 0x3FFL;
}

5
src/main/java/org/elasticsearch/common/bytes/HashedBytesArray.java
View File

@ -22,7 +22,6 @@ package org.elasticsearch.common.bytes;
import com.google.common.base.Charsets;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.common.Unicode;
import org.elasticsearch.common.io.stream.BytesStreamInput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.jboss.netty.buffer.ChannelBuffer;
@ -47,10 +46,6 @@ public class HashedBytesArray implements BytesReference {
this.bytes = bytes;
}
public HashedBytesArray(String str) {
this(Unicode.fromStringAsBytes(str));
}
@Override
public byte get(int index) {
return bytes[index];

5
src/main/java/org/elasticsearch/common/compress/CompressedString.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.common.compress;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.Unicode;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.StreamInput;
@ -79,7 +78,7 @@ public class CompressedString implements Streamable {
}
public CompressedString(String str) throws IOException {
BytesRef result = Unicode.unsafeFromStringAsUtf8(str);
BytesRef result = new BytesRef(str);
this.bytes = CompressorFactory.defaultCompressor().compress(result.bytes, result.offset, result.length);
}
@ -93,7 +92,7 @@ public class CompressedString implements Streamable {
}
public String string() throws IOException {
return Unicode.fromBytes(uncompressed());
return new BytesRef(uncompressed()).utf8ToString();
}
public static CompressedString readCompressedString(StreamInput in) throws IOException {

4
src/main/java/org/elasticsearch/common/io/FastByteArrayOutputStream.java
View File

@ -27,6 +27,8 @@ import org.apache.lucene.util.ArrayUtil;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import com.google.common.base.Charsets;
/**
* Similar to {@link java.io.ByteArrayOutputStream} just not synced.
*/
@ -171,7 +173,7 @@ public class FastByteArrayOutputStream extends OutputStream implements BytesStre
* @since JDK1.1
*/
public String toString() {
return new String(buf, 0, count, Streams.UTF8);
return new String(buf, 0, count, Charsets.UTF_8);
}
/**

8
src/main/java/org/elasticsearch/common/io/Streams.java
View File

@ -23,6 +23,8 @@ import org.elasticsearch.common.Preconditions;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.CachedStreamOutput;
import com.google.common.base.Charsets;
import java.io.*;
import java.nio.charset.Charset;
@ -36,8 +38,6 @@ import java.nio.charset.Charset;
*/
public abstract class Streams {
public static final Charset UTF8 = Charset.forName("UTF-8");
public static final int BUFFER_SIZE = 1024 * 8;
@ -255,7 +255,7 @@ public abstract class Streams {
if (is == null) {
throw new FileNotFoundException("Resource [" + path + "] not found in classpath with class loader [" + classLoader + "]");
}
return copyToString(new InputStreamReader(is, UTF8));
return copyToString(new InputStreamReader(is, Charsets.UTF_8));
}
public static String copyToStringFromClasspath(String path) throws IOException {
@ -263,7 +263,7 @@ public abstract class Streams {
if (is == null) {
throw new FileNotFoundException("Resource [" + path + "] not found in classpath");
}
return copyToString(new InputStreamReader(is, UTF8));
return copyToString(new InputStreamReader(is, Charsets.UTF_8));
}
public static byte[] copyToBytesFromClasspath(String path) throws IOException {

4
src/main/java/org/elasticsearch/common/lucene/LoggerInfoStream.java
View File

@ -23,6 +23,8 @@ import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import com.google.common.base.Charsets;
import java.io.OutputStream;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
@ -74,7 +76,7 @@ public class LoggerInfoStream extends PrintStream {
* @throws UnsupportedEncodingException
*/
public LoggerInfoStream(ESLogger logger) throws UnsupportedEncodingException {
super((OutputStream) null, false, Streams.UTF8.name());
super((OutputStream) null, false, Charsets.UTF_8.name());
this.logger = logger;
}

4
src/main/java/org/elasticsearch/common/math/MathRuntimeException.java
View File

@ -31,6 +31,8 @@ import java.util.*;
import org.elasticsearch.common.io.Streams;
import com.google.common.base.Charsets;
/**
* Base class for commons-math unchecked exceptions.
*
@ -197,7 +199,7 @@ public class MathRuntimeException extends RuntimeException {
@Override
public void printStackTrace(final PrintStream out) {
synchronized (out) {
PrintWriter pw = new PrintWriter(new OutputStreamWriter(out, Streams.UTF8));
PrintWriter pw = new PrintWriter(new OutputStreamWriter(out, Charsets.UTF_8));
printStackTrace(pw);
// Flush the PrintWriter before it's GC'ed.
pw.flush();

3
src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.common.settings;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@ -747,7 +748,7 @@ public class ImmutableSettings implements Settings {
public Builder loadFromStream(String resourceName, InputStream is) throws SettingsException {
SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromResource(resourceName);
try {
Map<String, String> loadedSettings = settingsLoader.load(Streams.copyToString(new InputStreamReader(is, Streams.UTF8)));
Map<String, String> loadedSettings = settingsLoader.load(Streams.copyToString(new InputStreamReader(is, Charsets.UTF_8)));
put(loadedSettings);
} catch (Exception e) {
throw new SettingsException("Failed to load settings from [" + resourceName + "]", e);

4
src/main/java/org/elasticsearch/env/Environment.java vendored
View File

@ -23,6 +23,8 @@ import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.settings.Settings;
import com.google.common.base.Charsets;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
@ -164,7 +166,7 @@ public class Environment {
}
public String resolveConfigAndLoadToString(String path) throws FailedToResolveConfigException, IOException {
return Streams.copyToString(new InputStreamReader(resolveConfig(path).openStream(), Streams.UTF8));
return Streams.copyToString(new InputStreamReader(resolveConfig(path).openStream(), Charsets.UTF_8));
}
public URL resolveConfig(String path) throws FailedToResolveConfigException {

10
src/main/java/org/elasticsearch/index/cache/filter/support/CacheKeyFilter.java vendored
View File

@ -23,7 +23,7 @@ import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.Bits;
import org.elasticsearch.common.Unicode;
import org.elasticsearch.common.Strings;
import java.io.IOException;
import java.util.Arrays;
@ -43,17 +43,13 @@ public interface CacheKeyFilter {
}
public Key(String str) {
this(Unicode.fromStringAsBytes(str));
this(Strings.toUTF8Bytes(str));
}
public byte[] bytes() {
return this.bytes;
}
public String utf8ToString() {
return Unicode.fromBytes(bytes);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;

6
src/main/java/org/elasticsearch/index/cache/filter/weighted/WeightedFilterCache.java vendored
View File

@ -28,8 +28,10 @@ import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.docset.DocIdSets;
import org.elasticsearch.common.lucene.search.CachedFilter;
@ -93,9 +95,11 @@ public class WeightedFilterCache extends AbstractIndexComponent implements Filte
@Override
public void clear(String reason, String[] keys) {
logger.debug("clear keys [], reason [{}]", reason, keys);
final BytesRef spare = new BytesRef();
for (String key : keys) {
final byte[] keyBytes = Strings.toUTF8Bytes(key, spare);
for (Object readerKey : seenReaders.keySet()) {
indicesFilterCache.cache().invalidate(new FilterCacheKey(readerKey, new CacheKeyFilter.Key(key)));
indicesFilterCache.cache().invalidate(new FilterCacheKey(readerKey, new CacheKeyFilter.Key(keyBytes)));
}
}
}

5
src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java vendored
View File

@ -22,7 +22,9 @@ package org.elasticsearch.index.cache.id.simple;
import gnu.trove.impl.Constants;
import org.apache.lucene.index.*;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.UnicodeUtil;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.HashedBytesArray;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.inject.Inject;
@ -119,10 +121,11 @@ public class SimpleIdCache extends AbstractIndexComponent implements IdCache, Se
// We don't want to load uid of child documents, this allows us to not load uids of child types.
NavigableSet<HashedBytesArray> parentTypes = new TreeSet<HashedBytesArray>(UTF8SortedAsUnicodeComparator.utf8SortedAsUnicodeSortOrder);
BytesRef spare = new BytesRef();
for (String type : indexService.mapperService().types()) {
ParentFieldMapper parentFieldMapper = indexService.mapperService().documentMapper(type).parentFieldMapper();
if (parentFieldMapper != null) {
parentTypes.add(new HashedBytesArray(parentFieldMapper.type()));
parentTypes.add(new HashedBytesArray(Strings.toUTF8Bytes(parentFieldMapper.type(), spare)));
}
}

2
src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java
View File

@ -841,7 +841,7 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I
}
CheckIndex checkIndex = new CheckIndex(store.directory());
FastByteArrayOutputStream os = new FastByteArrayOutputStream();
PrintStream out = new PrintStream(os, false, Streams.UTF8.name());
PrintStream out = new PrintStream(os, false, Charsets.UTF_8.name());
checkIndex.setInfoStream(out);
out.flush();
CheckIndex.Status status = checkIndex.checkIndex();

4
src/main/java/org/elasticsearch/indices/cache/filter/terms/IndicesTermsFilterCache.java vendored
View File

@ -32,6 +32,7 @@ import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
@ -133,8 +134,9 @@ public class IndicesTermsFilterCache extends AbstractComponent {
}
public void clear(String reason, String[] keys) {
final BytesRef spare = new BytesRef();
for (String key : keys) {
cache.invalidate(new CacheKeyFilter.Key(key));
cache.invalidate(new CacheKeyFilter.Key(Strings.toUTF8Bytes(key, spare)));
}
}

3
src/main/java/org/elasticsearch/monitor/dump/AbstractDump.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.monitor.dump;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.io.Streams;
@ -89,6 +90,6 @@ public abstract class AbstractDump implements Dump {
@Override
public Writer createFileWriter(String name) throws DumpException {
return new OutputStreamWriter(createFileOutputStream(name), Streams.UTF8);
return new OutputStreamWriter(createFileOutputStream(name), Charsets.UTF_8);
}
}

3
src/main/java/org/elasticsearch/script/ScriptService.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.script;
import com.google.common.base.Charsets;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.ImmutableMap;
@ -121,7 +122,7 @@ public class ScriptService extends AbstractComponent {
if (s.equals(ext)) {
found = true;
try {
String script = Streams.copyToString(new InputStreamReader(new FileInputStream(file), Streams.UTF8));
String script = Streams.copyToString(new InputStreamReader(new FileInputStream(file), Charsets.UTF_8));
staticCache.put(scriptName, new CompiledScript(engineService.types()[0], engineService.compile(script)));
} catch (Exception e) {
logger.warn("failed to load/compile script [{}]", e, scriptName);

6
src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.search.builder;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import gnu.trove.iterator.TObjectFloatIterator;
@ -26,7 +27,6 @@ import gnu.trove.map.hash.TObjectFloatHashMap;
import org.elasticsearch.ElasticSearchGenerationException;
import org.elasticsearch.client.Requests;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Unicode;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.unit.TimeValue;
@ -156,7 +156,7 @@ public class SearchSourceBuilder implements ToXContent {
* Constructs a new search source builder with a raw search query.
*/
public SearchSourceBuilder query(String queryString) {
return query(Unicode.fromStringAsBytes(queryString));
return query(queryString.getBytes(Charsets.UTF_8));
}
/**
@ -193,7 +193,7 @@ public class SearchSourceBuilder implements ToXContent {
* (and not facets for example).
*/
public SearchSourceBuilder filter(String filterString) {
return filter(Unicode.fromStringAsBytes(filterString));
return filter(filterString.getBytes(Charsets.UTF_8));
}
/**

3
src/main/java/org/elasticsearch/search/facet/datehistogram/InternalCountDateHistogramFacet.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.search.facet.datehistogram;
import gnu.trove.iterator.TLongLongIterator;
import gnu.trove.map.hash.TLongLongHashMap;
import org.elasticsearch.common.CacheRecycler;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.HashedBytesArray;
import org.elasticsearch.common.io.stream.StreamInput;
@ -40,7 +41,7 @@ import java.util.List;
*/
public class InternalCountDateHistogramFacet extends InternalDateHistogramFacet {
private static final BytesReference STREAM_TYPE = new HashedBytesArray("cdHistogram");
private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("cdHistogram"));
public static void registerStreams() {
Streams.registerStream(STREAM, STREAM_TYPE);

3
src/main/java/org/elasticsearch/search/facet/datehistogram/InternalFullDateHistogramFacet.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.search.facet.datehistogram;
import org.elasticsearch.common.CacheRecycler;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.HashedBytesArray;
import org.elasticsearch.common.io.stream.StreamInput;
@ -37,7 +38,7 @@ import java.util.*;
*/
public class InternalFullDateHistogramFacet extends InternalDateHistogramFacet {
private static final BytesReference STREAM_TYPE = new HashedBytesArray("fdHistogram");
private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("fdHistogram"));
public static void registerStreams() {
Streams.registerStream(STREAM, STREAM_TYPE);

3
src/main/java/org/elasticsearch/search/facet/filter/InternalFilterFacet.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.search.facet.filter;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.HashedBytesArray;
import org.elasticsearch.common.io.stream.StreamInput;
@ -36,7 +37,7 @@ import java.util.List;
*/
public class InternalFilterFacet extends InternalFacet implements FilterFacet {
private static final BytesReference STREAM_TYPE = new HashedBytesArray("filter");
private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("filter"));
public static void registerStreams() {
Streams.registerStream(STREAM, STREAM_TYPE);

4
src/main/java/org/elasticsearch/search/facet/geodistance/InternalGeoDistanceFacet.java
View File

@ -20,6 +20,8 @@
package org.elasticsearch.search.facet.geodistance;
import com.google.common.collect.ImmutableList;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.HashedBytesArray;
import org.elasticsearch.common.io.stream.StreamInput;
@ -38,7 +40,7 @@ import java.util.List;
*/
public class InternalGeoDistanceFacet extends InternalFacet implements GeoDistanceFacet {
private static final BytesReference STREAM_TYPE = new HashedBytesArray("geoDistance");
private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("geoDistance"));
public static void registerStreams() {
Streams.registerStream(STREAM, STREAM_TYPE);

3
src/main/java/org/elasticsearch/search/facet/histogram/InternalCountHistogramFacet.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.search.facet.histogram;
import gnu.trove.iterator.TLongLongIterator;
import gnu.trove.map.hash.TLongLongHashMap;
import org.elasticsearch.common.CacheRecycler;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.HashedBytesArray;
import org.elasticsearch.common.io.stream.StreamInput;
@ -40,7 +41,7 @@ import java.util.List;
*/
public class InternalCountHistogramFacet extends InternalHistogramFacet {
private static final BytesReference STREAM_TYPE = new HashedBytesArray("cHistogram");
private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("cHistogram"));
public static void registerStreams() {
Streams.registerStream(STREAM, STREAM_TYPE);

3
src/main/java/org/elasticsearch/search/facet/histogram/InternalFullHistogramFacet.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.search.facet.histogram;
import org.elasticsearch.common.CacheRecycler;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.HashedBytesArray;
import org.elasticsearch.common.io.stream.StreamInput;
@ -37,7 +38,7 @@ import java.util.*;
*/
public class InternalFullHistogramFacet extends InternalHistogramFacet {
private static final BytesReference STREAM_TYPE = new HashedBytesArray("fHistogram");
private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("fHistogram"));
public static void registerStreams() {
Streams.registerStream(STREAM, STREAM_TYPE);

3
src/main/java/org/elasticsearch/search/facet/query/InternalQueryFacet.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.search.facet.query;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.HashedBytesArray;
import org.elasticsearch.common.io.stream.StreamInput;
@ -36,7 +37,7 @@ import java.util.List;
*/
public class InternalQueryFacet extends InternalFacet implements QueryFacet {
private static final BytesReference STREAM_TYPE = new HashedBytesArray("query");
private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("query"));
public static void registerStreams() {
Streams.registerStream(STREAM, STREAM_TYPE);

4
src/main/java/org/elasticsearch/search/facet/range/InternalRangeFacet.java
View File

@ -20,6 +20,8 @@
package org.elasticsearch.search.facet.range;
import com.google.common.collect.ImmutableList;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.HashedBytesArray;
import org.elasticsearch.common.io.stream.StreamInput;
@ -38,7 +40,7 @@ import java.util.List;
*/
public class InternalRangeFacet extends InternalFacet implements RangeFacet {
private static final BytesReference STREAM_TYPE = new HashedBytesArray("range");
private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("range"));
public static void registerStreams() {
Streams.registerStream(STREAM, STREAM_TYPE);

3
src/main/java/org/elasticsearch/search/facet/statistical/InternalStatisticalFacet.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.search.facet.statistical;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.HashedBytesArray;
import org.elasticsearch.common.io.stream.StreamInput;
@ -36,7 +37,7 @@ import java.util.List;
*/
public class InternalStatisticalFacet extends InternalFacet implements StatisticalFacet {
private static final BytesReference STREAM_TYPE = new HashedBytesArray("statistical");
private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("statistical"));
public static void registerStreams() {
Streams.registerStream(STREAM, STREAM_TYPE);

3
src/main/java/org/elasticsearch/search/facet/terms/doubles/InternalDoubleTermsFacet.java
View File

@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList;
import gnu.trove.iterator.TDoubleIntIterator;
import gnu.trove.map.hash.TDoubleIntHashMap;
import org.elasticsearch.common.CacheRecycler;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.HashedBytesArray;
import org.elasticsearch.common.collect.BoundedTreeSet;
@ -47,7 +48,7 @@ import java.util.List;
*/
public class InternalDoubleTermsFacet extends InternalTermsFacet {
private static final BytesReference STREAM_TYPE = new HashedBytesArray("dTerms");
private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("dTerms"));
public static void registerStream() {
Streams.registerStream(STREAM, STREAM_TYPE);

3
src/main/java/org/elasticsearch/search/facet/terms/longs/InternalLongTermsFacet.java
View File

@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList;
import gnu.trove.iterator.TLongIntIterator;
import gnu.trove.map.hash.TLongIntHashMap;
import org.elasticsearch.common.CacheRecycler;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.HashedBytesArray;
import org.elasticsearch.common.collect.BoundedTreeSet;
@ -47,7 +48,7 @@ import java.util.List;
*/
public class InternalLongTermsFacet extends InternalTermsFacet {
private static final BytesReference STREAM_TYPE = new HashedBytesArray("lTerms");
private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("lTerms"));
public static void registerStream() {
Streams.registerStream(STREAM, STREAM_TYPE);

3
src/main/java/org/elasticsearch/search/facet/terms/strings/InternalStringTermsFacet.java
View File

@ -24,6 +24,7 @@ import gnu.trove.iterator.TObjectIntIterator;
import gnu.trove.map.hash.TObjectIntHashMap;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.CacheRecycler;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.HashedBytesArray;
@ -50,7 +51,7 @@ import java.util.List;
*/
public class InternalStringTermsFacet extends InternalTermsFacet {
private static final BytesReference STREAM_TYPE = new HashedBytesArray("tTerms");
private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("tTerms"));
public static void registerStream() {
Streams.registerStream(STREAM, STREAM_TYPE);

3
src/main/java/org/elasticsearch/search/facet/termsstats/doubles/InternalTermsStatsDoubleFacet.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.search.facet.termsstats.doubles;
import com.google.common.collect.ImmutableList;
import org.elasticsearch.common.CacheRecycler;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.HashedBytesArray;
import org.elasticsearch.common.io.stream.StreamInput;
@ -38,7 +39,7 @@ import java.util.*;
public class InternalTermsStatsDoubleFacet extends InternalTermsStatsFacet {
private static final BytesReference STREAM_TYPE = new HashedBytesArray("dTS");
private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("dTS"));
public static void registerStream() {
Streams.registerStream(STREAM, STREAM_TYPE);

3
src/main/java/org/elasticsearch/search/facet/termsstats/longs/InternalTermsStatsLongFacet.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.search.facet.termsstats.longs;
import com.google.common.collect.ImmutableList;
import org.elasticsearch.common.CacheRecycler;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.HashedBytesArray;
import org.elasticsearch.common.io.stream.StreamInput;
@ -38,7 +39,7 @@ import java.util.*;
public class InternalTermsStatsLongFacet extends InternalTermsStatsFacet {
private static final BytesReference STREAM_TYPE = new HashedBytesArray("lTS");
private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("lTS"));
public static void registerStream() {
Streams.registerStream(STREAM, STREAM_TYPE);

3
src/main/java/org/elasticsearch/search/facet/termsstats/strings/InternalTermsStatsStringFacet.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.search.facet.termsstats.strings;
import com.google.common.collect.ImmutableList;
import org.elasticsearch.common.CacheRecycler;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.HashedBytesArray;
@ -40,7 +41,7 @@ import java.util.*;
public class InternalTermsStatsStringFacet extends InternalTermsStatsFacet {
private static final BytesReference STREAM_TYPE = new HashedBytesArray("tTS");
private static final BytesReference STREAM_TYPE = new HashedBytesArray(Strings.toUTF8Bytes("tTS"));
public static void registerStream() {
Streams.registerStream(STREAM, STREAM_TYPE);

6
src/test/java/org/elasticsearch/test/integration/broadcast/BroadcastActionsTests.java
View File

@ -26,12 +26,14 @@ import org.elasticsearch.action.admin.indices.flush.FlushResponse;
import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
import org.elasticsearch.action.count.CountResponse;
import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading;
import org.elasticsearch.common.Unicode;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.test.integration.AbstractSharedClusterTest;
import org.testng.annotations.Test;
import com.beust.jcommander.Strings;
import com.google.common.base.Charsets;
import java.io.IOException;
import static org.elasticsearch.client.Requests.*;
@ -95,7 +97,7 @@ public class BroadcastActionsTests extends AbstractSharedClusterTest {
for (int i = 0; i < 5; i++) {
// test failed (simply query that can't be parsed)
CountResponse countResponse = client().count(countRequest("test").query(Unicode.fromStringAsBytes("{ term : { _type : \"type1 } }"))).actionGet();
CountResponse countResponse = client().count(countRequest("test").query("{ term : { _type : \"type1 } }".getBytes(Charsets.UTF_8))).actionGet();
assertThat(countResponse.getCount(), equalTo(0l));
assertThat(countResponse.getTotalShards(), equalTo(5));

28
src/test/java/org/elasticsearch/test/integration/document/DocumentActionsTests.java
View File

@ -19,6 +19,21 @@
package org.elasticsearch.test.integration.document;
import static org.elasticsearch.client.Requests.clearIndicesCacheRequest;
import static org.elasticsearch.client.Requests.clusterHealthRequest;
import static org.elasticsearch.client.Requests.countRequest;
import static org.elasticsearch.client.Requests.getRequest;
import static org.elasticsearch.client.Requests.indexRequest;
import static org.elasticsearch.client.Requests.refreshRequest;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
import java.io.IOException;
import java.util.Map;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse;
@ -33,22 +48,13 @@ import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading;
import org.elasticsearch.action.support.replication.ReplicationType;
import org.elasticsearch.common.Unicode;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.test.integration.AbstractSharedClusterTest;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.io.IOException;
import java.util.Map;
import static org.elasticsearch.client.Requests.*;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
import com.google.common.base.Charsets;
/**
*
@ -196,7 +202,7 @@ public class DocumentActionsTests extends AbstractSharedClusterTest {
assertThat(countResponse.getFailedShards(), equalTo(0));
// test failed (simply query that can't be parsed)
countResponse = client().count(countRequest("test").query(Unicode.fromStringAsBytes("{ term : { _type : \"type1 } }"))).actionGet();
countResponse = client().count(countRequest("test").query("{ term : { _type : \"type1 } }".getBytes(Charsets.UTF_8))).actionGet();
assertThat(countResponse.getCount(), equalTo(0l));
assertThat(countResponse.getSuccessfulShards(), equalTo(0));

6
src/test/java/org/elasticsearch/test/integration/rest/helper/HttpClient.java
View File

@ -23,6 +23,8 @@ import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.transport.TransportAddress;
import com.google.common.base.Charsets;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@ -96,7 +98,7 @@ public class HttpClient {
InputStream inputStream = urlConnection.getInputStream();
String body = null;
try {
body = Streams.copyToString(new InputStreamReader(inputStream, Streams.UTF8));
body = Streams.copyToString(new InputStreamReader(inputStream, Charsets.UTF_8));
} catch (IOException e1) {
throw new ElasticSearchException("problem reading error stream", e1);
}
@ -105,7 +107,7 @@ public class HttpClient {
InputStream errStream = urlConnection.getErrorStream();
String body = null;
try {
body = Streams.copyToString(new InputStreamReader(errStream, Streams.UTF8));
body = Streams.copyToString(new InputStreamReader(errStream, Charsets.UTF_8));
} catch (IOException e1) {
throw new ElasticSearchException("problem reading error stream", e1);
}

8
src/test/java/org/elasticsearch/test/integration/search/basic/TransportSearchFailuresTests.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.test.integration.search.basic;
import static org.elasticsearch.client.Requests.clusterHealthRequest;
import static org.elasticsearch.client.Requests.createIndexRequest;
import static org.elasticsearch.client.Requests.refreshRequest;
import static org.elasticsearch.client.Requests.searchRequest;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
@ -41,11 +40,12 @@ import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.Unicode;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.test.integration.AbstractSharedClusterTest;
import org.testng.annotations.Test;
import com.google.common.base.Charsets;
/**
*
*/
@ -69,7 +69,7 @@ public class TransportSearchFailuresTests extends AbstractSharedClusterTest {
assertThat(refreshResponse.getFailedShards(), equalTo(0));
for (int i = 0; i < 5; i++) {
try {
SearchResponse searchResponse = client().search(searchRequest("test").source(Unicode.fromStringAsBytes("{ xxx }"))).actionGet();
SearchResponse searchResponse = client().search(searchRequest("test").source("{ xxx }".getBytes(Charsets.UTF_8))).actionGet();
assertThat(searchResponse.getTotalShards(), equalTo(3));
assertThat(searchResponse.getSuccessfulShards(), equalTo(0));
assertThat(searchResponse.getFailedShards(), equalTo(3));
@ -98,7 +98,7 @@ public class TransportSearchFailuresTests extends AbstractSharedClusterTest {
for (int i = 0; i < 5; i++) {
try {
SearchResponse searchResponse = client().search(searchRequest("test").source(Unicode.fromStringAsBytes("{ xxx }"))).actionGet();
SearchResponse searchResponse = client().search(searchRequest("test").source("{ xxx }".getBytes(Charsets.UTF_8))).actionGet();
assertThat(searchResponse.getTotalShards(), equalTo(3));
assertThat(searchResponse.getSuccessfulShards(), equalTo(0));
assertThat(searchResponse.getFailedShards(), equalTo(3));

4
src/test/java/org/elasticsearch/test/integration/search/basic/TransportTwoNodesSearchTests.java
View File

@ -50,7 +50,6 @@ import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.Unicode;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.Scroll;
@ -64,6 +63,7 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import com.google.common.base.Charsets;
import com.google.common.collect.Sets;
/**
@ -339,7 +339,7 @@ public class TransportTwoNodesSearchTests extends AbstractSharedClusterTest {
public void testFailedSearchWithWrongQuery() throws Exception {
logger.info("Start Testing failed search with wrong query");
try {
SearchResponse searchResponse = client().search(searchRequest("test").source(Unicode.fromStringAsBytes("{ xxx }"))).actionGet();
SearchResponse searchResponse = client().search(searchRequest("test").source("{ xxx }".getBytes(Charsets.UTF_8))).actionGet();
assertThat(searchResponse.getTotalShards(), equalTo(3));
assertThat(searchResponse.getSuccessfulShards(), equalTo(0));
assertThat(searchResponse.getFailedShards(), equalTo(3));

6
src/test/java/org/elasticsearch/test/integration/search/suggest/SuggestSearchTests.java
View File

@ -56,6 +56,8 @@ import org.elasticsearch.test.integration.AbstractSharedClusterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.common.base.Charsets;
/**
*/
public class SuggestSearchTests extends AbstractSharedClusterTest {
@ -550,7 +552,7 @@ public class SuggestSearchTests extends AbstractSharedClusterTest {
client().admin().indices().prepareCreate("test").setSettings(builder.build()).addMapping("type1", mapping).execute().actionGet();
client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet();
BufferedReader reader = new BufferedReader(new InputStreamReader(SuggestSearchTests.class.getResourceAsStream("/config/names.txt"), Streams.UTF8));
BufferedReader reader = new BufferedReader(new InputStreamReader(SuggestSearchTests.class.getResourceAsStream("/config/names.txt"), Charsets.UTF_8));
String line = null;
while ((line = reader.readLine()) != null) {
client().prepareIndex("test", "type1")
@ -884,7 +886,7 @@ public class SuggestSearchTests extends AbstractSharedClusterTest {
client().admin().indices().prepareCreate("test").setSettings(builder.build()).addMapping("type1", mapping).execute().actionGet();
client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet();
BufferedReader reader = new BufferedReader(new InputStreamReader(SuggestSearchTests.class.getResourceAsStream("/config/names.txt"), Streams.UTF8));
BufferedReader reader = new BufferedReader(new InputStreamReader(SuggestSearchTests.class.getResourceAsStream("/config/names.txt"), Charsets.UTF_8));
String line = null;
while ((line = reader.readLine()) != null) {
client().prepareIndex("test", "type1")

8
src/test/java/org/elasticsearch/test/integration/validate/SimpleValidateQueryTests.java
View File

@ -35,6 +35,8 @@ import org.hamcrest.Matcher;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
import com.google.common.base.Charsets;
import java.io.IOException;
import static org.hamcrest.MatcherAssert.assertThat;
@ -60,7 +62,7 @@ public class SimpleValidateQueryTests extends AbstractSharedClusterTest {
client().admin().indices().prepareRefresh().execute().actionGet();
assertThat(client().admin().indices().prepareValidateQuery("test").setQuery("foo".getBytes(Streams.UTF8)).execute().actionGet().isValid(), equalTo(false));
assertThat(client().admin().indices().prepareValidateQuery("test").setQuery("foo".getBytes(Charsets.UTF_8)).execute().actionGet().isValid(), equalTo(false));
assertThat(client().admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryString("_id:1")).execute().actionGet().isValid(), equalTo(true));
assertThat(client().admin().indices().prepareValidateQuery("test").setQuery(QueryBuilders.queryString("_i:d:1")).execute().actionGet().isValid(), equalTo(false));
@ -100,7 +102,7 @@ public class SimpleValidateQueryTests extends AbstractSharedClusterTest {
ValidateQueryResponse response;
response = client().admin().indices().prepareValidateQuery("test")
.setQuery("foo".getBytes(Streams.UTF8))
.setQuery("foo".getBytes(Charsets.UTF_8))
.setExplain(true)
.execute().actionGet();
assertThat(response.isValid(), equalTo(false));
@ -207,7 +209,7 @@ public class SimpleValidateQueryTests extends AbstractSharedClusterTest {
for (Client client : cluster().clients()) {
ValidateQueryResponse response = client.admin().indices().prepareValidateQuery("test")
.setQuery("foo".getBytes(Streams.UTF8))
.setQuery("foo".getBytes(Charsets.UTF_8))
.setExplain(true)
.execute().actionGet();
assertThat(response.isValid(), equalTo(false));

10
src/test/java/org/elasticsearch/test/unit/action/bulk/BulkRequestTests.java
View File

@ -27,6 +27,8 @@ import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.io.Streams;
import org.testng.annotations.Test;
import com.google.common.base.Charsets;
import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
@ -38,7 +40,7 @@ public class BulkRequestTests {
public void testSimpleBulk1() throws Exception {
String bulkAction = copyToStringFromClasspath("/org/elasticsearch/test/unit/action/bulk/simple-bulk.json");
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.add(bulkAction.getBytes(Streams.UTF8), 0, bulkAction.length(), true, null, null);
bulkRequest.add(bulkAction.getBytes(Charsets.UTF_8), 0, bulkAction.length(), true, null, null);
assertThat(bulkRequest.numberOfActions(), equalTo(3));
assertThat(((IndexRequest) bulkRequest.requests().get(0)).source().toBytes(), equalTo(new BytesArray("{ \"field1\" : \"value1\" }").toBytes()));
assertThat(bulkRequest.requests().get(1), instanceOf(DeleteRequest.class));
@ -49,7 +51,7 @@ public class BulkRequestTests {
public void testSimpleBulk2() throws Exception {
String bulkAction = copyToStringFromClasspath("/org/elasticsearch/test/unit/action/bulk/simple-bulk2.json");
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.add(bulkAction.getBytes(Streams.UTF8), 0, bulkAction.length(), true, null, null);
bulkRequest.add(bulkAction.getBytes(Charsets.UTF_8), 0, bulkAction.length(), true, null, null);
assertThat(bulkRequest.numberOfActions(), equalTo(3));
}
@ -57,7 +59,7 @@ public class BulkRequestTests {
public void testSimpleBulk3() throws Exception {
String bulkAction = copyToStringFromClasspath("/org/elasticsearch/test/unit/action/bulk/simple-bulk3.json");
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.add(bulkAction.getBytes(Streams.UTF8), 0, bulkAction.length(), true, null, null);
bulkRequest.add(bulkAction.getBytes(Charsets.UTF_8), 0, bulkAction.length(), true, null, null);
assertThat(bulkRequest.numberOfActions(), equalTo(3));
}
@ -65,7 +67,7 @@ public class BulkRequestTests {
public void testSimpleBulk4() throws Exception {
String bulkAction = copyToStringFromClasspath("/org/elasticsearch/test/unit/action/bulk/simple-bulk4.json");
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.add(bulkAction.getBytes(Streams.UTF8), 0, bulkAction.length(), true, null, null);
bulkRequest.add(bulkAction.getBytes(Charsets.UTF_8), 0, bulkAction.length(), true, null, null);
assertThat(bulkRequest.numberOfActions(), equalTo(4));
assertThat(((UpdateRequest) bulkRequest.requests().get(0)).id(), equalTo("1"));
assertThat(((UpdateRequest) bulkRequest.requests().get(0)).retryOnConflict(), equalTo(2));

27
src/test/java/org/elasticsearch/test/unit/common/io/StreamsTests.java
View File

@ -19,16 +19,23 @@
package org.elasticsearch.test.unit.common.io;
import org.elasticsearch.common.io.Streams;
import org.testng.annotations.Test;
import java.io.*;
import java.util.Arrays;
import static org.elasticsearch.common.io.Streams.*;
import static org.elasticsearch.common.io.Streams.copy;
import static org.elasticsearch.common.io.Streams.copyToByteArray;
import static org.elasticsearch.common.io.Streams.copyToString;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.StringReader;
import java.io.StringWriter;
import java.util.Arrays;
import org.testng.annotations.Test;
import com.google.common.base.Charsets;
/**
* Unit tests for {@link org.elasticsearch.common.io.Streams}.
*
@ -38,7 +45,7 @@ public class StreamsTests {
@Test
public void testCopyFromInputStream() throws IOException {
byte[] content = "content".getBytes(Streams.UTF8);
byte[] content = "content".getBytes(Charsets.UTF_8);
ByteArrayInputStream in = new ByteArrayInputStream(content);
ByteArrayOutputStream out = new ByteArrayOutputStream(content.length);
long count = copy(in, out);
@ -49,7 +56,7 @@ public class StreamsTests {
@Test
public void testCopyFromByteArray() throws IOException {
byte[] content = "content".getBytes(Streams.UTF8);
byte[] content = "content".getBytes(Charsets.UTF_8);
ByteArrayOutputStream out = new ByteArrayOutputStream(content.length);
copy(content, out);
assertThat(Arrays.equals(content, out.toByteArray()), equalTo(true));
@ -57,7 +64,7 @@ public class StreamsTests {
@Test
public void testCopyToByteArray() throws IOException {
byte[] content = "content".getBytes(Streams.UTF8);
byte[] content = "content".getBytes(Charsets.UTF_8);
ByteArrayInputStream in = new ByteArrayInputStream(content);
byte[] result = copyToByteArray(in);
assertThat(Arrays.equals(content, result), equalTo(true));

4
src/test/java/org/elasticsearch/test/unit/index/analysis/AnalysisModuleTests.java
View File

@ -45,6 +45,8 @@ import org.elasticsearch.test.unit.index.analysis.filter1.MyFilterTokenFilterFac
import org.hamcrest.MatcherAssert;
import org.testng.annotations.Test;
import com.google.common.base.Charsets;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
@ -172,7 +174,7 @@ public class AnalysisModuleTests {
BufferedWriter writer = null;
try {
writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(wordListFile), Streams.UTF8));
writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(wordListFile), Charsets.UTF_8));
for (String word : words) {
writer.write(word);
writer.write('\n');

23
src/test/java/org/elasticsearch/test/unit/index/cache/id/SimpleIdCacheTests.java vendored
View File

@ -30,6 +30,7 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.RAMDirectory;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.HashedBytesArray;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.Injector;
@ -132,10 +133,10 @@ public class SimpleIdCacheTests {
assertThat(typeCache.parentIdByDoc(3).toUtf8(), equalTo("1"));
assertThat(typeCache.parentIdByDoc(4).toUtf8(), equalTo("1"));
assertThat(typeCache.docById(new HashedBytesArray("1")), equalTo(0));
assertThat(typeCache.docById(new HashedBytesArray("2")), equalTo(-1));
assertThat(typeCache.docById(new HashedBytesArray("3")), equalTo(-1));
assertThat(typeCache.docById(new HashedBytesArray("4")), equalTo(-1));
assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("1"))), equalTo(0));
assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("2"))), equalTo(-1));
assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("3"))), equalTo(-1));
assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("4"))), equalTo(-1));
// Verify simple id cache for segment 2
readerCache = idCache.reader(leaves.get(1).reader());
@ -153,10 +154,10 @@ public class SimpleIdCacheTests {
assertThat(typeCache.parentIdByDoc(3), nullValue());
assertThat(typeCache.parentIdByDoc(4), nullValue());
assertThat(typeCache.docById(new HashedBytesArray("2")), equalTo(1));
assertThat(typeCache.docById(new HashedBytesArray("3")), equalTo(2));
assertThat(typeCache.docById(new HashedBytesArray("4")), equalTo(3));
assertThat(typeCache.docById(new HashedBytesArray("5")), equalTo(4));
assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("2"))), equalTo(1));
assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("3"))), equalTo(2));
assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("4"))), equalTo(3));
assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("5"))), equalTo(4));
// Verify simple id cache for segment 3
readerCache = idCache.reader(leaves.get(2).reader());
@ -176,8 +177,8 @@ public class SimpleIdCacheTests {
assertThat(typeCache.parentIdByDoc(4).toUtf8(), equalTo("4"));
assertThat(typeCache.parentIdByDoc(5), nullValue());
assertThat(typeCache.docById(new HashedBytesArray("6")), equalTo(0));
assertThat(typeCache.docById(new HashedBytesArray("7")), equalTo(5));
assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("6"))), equalTo(0));
assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("7"))), equalTo(5));
// Verify simple id cache for segment 4
readerCache = idCache.reader(leaves.get(3).reader());
@ -195,7 +196,7 @@ public class SimpleIdCacheTests {
assertThat(typeCache.parentIdByDoc(3), nullValue());
assertThat(typeCache.parentIdByDoc(4), nullValue());
assertThat(typeCache.docById(new HashedBytesArray("8")), equalTo(4));
assertThat(typeCache.docById(new HashedBytesArray(Strings.toUTF8Bytes("8"))), equalTo(4));
}
@Test(expectedExceptions = AssertionError.class)

3
src/test/java/org/elasticsearch/test/unit/index/gateway/CommitPointsTests.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.test.unit.index.gateway;
import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import org.elasticsearch.common.io.Streams;
@ -54,7 +55,7 @@ public class CommitPointsTests {
CommitPoint commitPoint = new CommitPoint(1, "test", CommitPoint.Type.GENERATED, indexFiles, translogFiles);
byte[] serialized = CommitPoints.toXContent(commitPoint);
logger.info("serialized commit_point {}", new String(serialized, Streams.UTF8));
logger.info("serialized commit_point {}", new String(serialized, Charsets.UTF_8));
CommitPoint desCp = CommitPoints.fromXContent(serialized);
assertThat(desCp.version(), equalTo(commitPoint.version()));

4
src/test/java/org/elasticsearch/test/unit/index/mapper/simple/SimpleMapperTests.java
View File

@ -30,6 +30,8 @@ import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.test.unit.index.mapper.MapperTests;
import org.testng.annotations.Test;
import com.google.common.base.Charsets;
import static org.elasticsearch.common.io.Streams.copyToBytesFromClasspath;
import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath;
import static org.elasticsearch.index.mapper.MapperBuilders.*;
@ -124,7 +126,7 @@ public class SimpleMapperTests {
.add(object("name").add(stringField("first").store(true).index(false)))
).build(mapperParser);
BytesReference json = new BytesArray("".getBytes(Streams.UTF8));
BytesReference json = new BytesArray("".getBytes(Charsets.UTF_8));
try {
docMapper.parse("person", "1", json).rootDoc();
assertThat("this point is never reached", false);

3
src/test/java/org/elasticsearch/test/unit/index/query/SimpleIndexQueryParserTests.java
View File

@ -31,6 +31,7 @@ import org.apache.lucene.spatial.prefix.IntersectsPrefixTreeFilter;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Injector;
@ -759,7 +760,7 @@ public class SimpleIndexQueryParserTests {
Filter filter = ((XFilteredQuery) parsedQuery.query()).getFilter();
assertThat(filter, instanceOf(CacheKeyFilter.Wrapper.class));
CacheKeyFilter.Wrapper wrapper = (CacheKeyFilter.Wrapper) filter;
assertThat(wrapper.cacheKey().utf8ToString(), equalTo("key"));
assertThat(new BytesRef(wrapper.cacheKey().bytes()).utf8ToString(), equalTo("key"));
assertThat(wrapper.wrappedFilter(), instanceOf(RegexpFilter.class));
RegexpFilter regexpFilter = (RegexpFilter) wrapper.wrappedFilter();
assertThat(regexpFilter.field(), equalTo("name.first"));

8
src/test/java/org/elasticsearch/test/unit/search/suggest/phrase/NoisyChannelSpellCheckerTests.java
View File

@ -61,6 +61,8 @@ import org.elasticsearch.search.suggest.phrase.NoisyChannelSpellChecker;
import org.elasticsearch.search.suggest.phrase.StupidBackoffScorer;
import org.elasticsearch.search.suggest.phrase.WordScorer;
import org.testng.annotations.Test;
import com.google.common.base.Charsets;
public class NoisyChannelSpellCheckerTests {
@Test
@ -94,7 +96,7 @@ public class NoisyChannelSpellCheckerTests {
IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_41, wrapper);
IndexWriter writer = new IndexWriter(dir, conf);
BufferedReader reader = new BufferedReader(new InputStreamReader(NoisyChannelSpellCheckerTests.class.getResourceAsStream("/config/names.txt"), Streams.UTF8));
BufferedReader reader = new BufferedReader(new InputStreamReader(NoisyChannelSpellCheckerTests.class.getResourceAsStream("/config/names.txt"), Charsets.UTF_8));
String line = null;
while ((line = reader.readLine()) != null) {
Document doc = new Document();
@ -205,7 +207,7 @@ public class NoisyChannelSpellCheckerTests {
IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_41, wrapper);
IndexWriter writer = new IndexWriter(dir, conf);
BufferedReader reader = new BufferedReader(new InputStreamReader(NoisyChannelSpellCheckerTests.class.getResourceAsStream("/config/names.txt"), Streams.UTF8));
BufferedReader reader = new BufferedReader(new InputStreamReader(NoisyChannelSpellCheckerTests.class.getResourceAsStream("/config/names.txt"), Charsets.UTF_8));
String line = null;
while ((line = reader.readLine()) != null) {
Document doc = new Document();
@ -290,7 +292,7 @@ public class NoisyChannelSpellCheckerTests {
IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_41, wrapper);
IndexWriter writer = new IndexWriter(dir, conf);
BufferedReader reader = new BufferedReader(new InputStreamReader(NoisyChannelSpellCheckerTests.class.getResourceAsStream("/config/names.txt"), Streams.UTF8));
BufferedReader reader = new BufferedReader(new InputStreamReader(NoisyChannelSpellCheckerTests.class.getResourceAsStream("/config/names.txt"), Charsets.UTF_8));
String line = null;
while ((line = reader.readLine()) != null) {
Document doc = new Document();