mapping = new HashMap<>(numEntries);
for (int i = 0; i < numEntries; i++) {
@@ -141,8 +143,10 @@ public final class CompoundFileDirectory extends BaseDirectory {
fileEntry.offset = entriesStream.readLong();
fileEntry.length = entriesStream.readLong();
}
- if (entriesStream.getFilePointer() != entriesStream.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + entriesFileName + "\": read " + entriesStream.getFilePointer() + " vs size " + entriesStream.length() + " (resource: " + entriesStream + ")");
+ if (version >= CompoundFileWriter.VERSION_CHECKSUM) {
+ CodecUtil.checkFooter(entriesStream);
+ } else {
+ CodecUtil.checkEOF(entriesStream);
}
return mapping;
} catch (IOException ioe) {
diff --git a/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java b/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java
index aefe2a46d74..405bf973c90 100644
--- a/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java
@@ -54,7 +54,8 @@ final class CompoundFileWriter implements Closeable{
// versioning for the .cfs file
static final String DATA_CODEC = "CompoundFileWriterData";
static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
+ static final int VERSION_CHECKSUM = 1;
+ static final int VERSION_CURRENT = VERSION_CHECKSUM;
// versioning for the .cfe file
static final String ENTRY_CODEC = "CompoundFileWriterEntries";
@@ -140,6 +141,7 @@ final class CompoundFileWriter implements Closeable{
// open the compound stream
getOutput();
assert dataOut != null;
+ CodecUtil.writeFooter(dataOut);
} catch (IOException e) {
priorException = e;
} finally {
@@ -202,6 +204,7 @@ final class CompoundFileWriter implements Closeable{
entryOut.writeLong(fe.offset);
entryOut.writeLong(fe.length);
}
+ CodecUtil.writeFooter(entryOut);
}
IndexOutput createOutput(String name, IOContext context) throws IOException {
@@ -342,6 +345,11 @@ final class CompoundFileWriter implements Closeable{
writtenBytes += length;
delegate.writeBytes(b, offset, length);
}
+
+ @Override
+ public long getChecksum() throws IOException {
+ return delegate.getChecksum();
+ }
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/store/DataInput.java b/lucene/core/src/java/org/apache/lucene/store/DataInput.java
index 6bcf18ee523..adbe38c1ada 100644
--- a/lucene/core/src/java/org/apache/lucene/store/DataInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/DataInput.java
@@ -18,13 +18,12 @@ package org.apache.lucene.store;
*/
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-import org.apache.lucene.util.IOUtils;
-
/**
* Abstract base class for performing read operations of Lucene's low-level
* data types.
@@ -188,7 +187,7 @@ public abstract class DataInput implements Cloneable {
int length = readVInt();
final byte[] bytes = new byte[length];
readBytes(bytes, 0, length);
- return new String(bytes, 0, length, IOUtils.CHARSET_UTF_8);
+ return new String(bytes, 0, length, StandardCharsets.UTF_8);
}
/** Returns a clone of this stream.
diff --git a/lucene/core/src/java/org/apache/lucene/store/Directory.java b/lucene/core/src/java/org/apache/lucene/store/Directory.java
index 9af6ff04cbc..f45abb1ee3d 100644
--- a/lucene/core/src/java/org/apache/lucene/store/Directory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/Directory.java
@@ -100,7 +100,12 @@ public abstract class Directory implements Closeable {
* Throws {@link FileNotFoundException} or {@link NoSuchFileException}
* if the file does not exist.
*/
- public abstract IndexInput openInput(String name, IOContext context) throws IOException;
+ public abstract IndexInput openInput(String name, IOContext context) throws IOException;
+
+ /** Returns a stream reading an existing file, computing checksum as it reads */
+ public ChecksumIndexInput openChecksumInput(String name, IOContext context) throws IOException {
+ return new BufferedChecksumIndexInput(openInput(name, context));
+ }
/** Construct a {@link Lock}.
* @param name the name of the lock file
diff --git a/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java b/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java
index 5fec7b596ae..b9196a1d8f5 100644
--- a/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java
@@ -43,6 +43,8 @@ public abstract class IndexOutput extends DataOutput implements Closeable {
*/
public abstract long getFilePointer();
+ /** Returns the current checksum of bytes written so far */
+ public abstract long getChecksum() throws IOException;
/** The number of bytes in the file. */
public abstract long length() throws IOException;
diff --git a/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java b/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java
index 67f39ef3971..1f0c74f78d7 100644
--- a/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java
+++ b/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java
@@ -18,6 +18,8 @@ package org.apache.lucene.store;
*/
import java.io.IOException;
+import java.util.zip.CRC32;
+import java.util.zip.Checksum;
/**
* A memory-resident {@link IndexOutput} implementation.
@@ -35,6 +37,8 @@ public class RAMOutputStream extends IndexOutput {
private int bufferPosition;
private long bufferStart;
private int bufferLength;
+
+ private Checksum crc = new BufferedChecksum(new CRC32());
/** Construct an empty output buffer. */
public RAMOutputStream() {
@@ -95,6 +99,7 @@ public class RAMOutputStream extends IndexOutput {
bufferStart = 0;
bufferLength = 0;
file.setLength(0);
+ crc.reset();
}
@Override
@@ -113,12 +118,14 @@ public class RAMOutputStream extends IndexOutput {
currentBufferIndex++;
switchCurrentBuffer();
}
+ crc.update(b);
currentBuffer[bufferPosition++] = b;
}
@Override
public void writeBytes(byte[] b, int offset, int len) throws IOException {
assert b != null;
+ crc.update(b, offset, len);
while (len > 0) {
if (bufferPosition == bufferLength) {
currentBufferIndex++;
@@ -165,5 +172,10 @@ public class RAMOutputStream extends IndexOutput {
/** Returns byte usage of all buffers. */
public long sizeInBytes() {
return (long) file.numBuffers() * (long) BUFFER_SIZE;
- }
+ }
+
+ @Override
+ public long getChecksum() throws IOException {
+ return crc.getValue();
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/util/IOUtils.java b/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
index 52cbf098d5b..2e183c42348 100644
--- a/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
+++ b/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
@@ -31,23 +31,29 @@ import java.io.Reader;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
/** This class emulates the new Java 7 "Try-With-Resources" statement.
* Remove once Lucene is on Java 7.
* @lucene.internal */
public final class IOUtils {
- /**
- * UTF-8 charset string
- * @see Charset#forName(String)
- */
- public static final String UTF_8 = "UTF-8";
-
/**
* UTF-8 {@link Charset} instance to prevent repeated
* {@link Charset#forName(String)} lookups
+ * @deprecated Use {@link StandardCharsets#UTF_8} instead.
*/
- public static final Charset CHARSET_UTF_8 = Charset.forName("UTF-8");
+ @Deprecated
+ public static final Charset CHARSET_UTF_8 = StandardCharsets.UTF_8;
+
+ /**
+ * UTF-8 charset string.
+ *
Where possible, use {@link StandardCharsets#UTF_8} instead,
+ * as using the String constant may slow things down.
+ * @see StandardCharsets#UTF_8
+ */
+ public static final String UTF_8 = StandardCharsets.UTF_8.name();
+
private IOUtils() {} // no instance
/**
diff --git a/lucene/core/src/java/org/apache/lucene/util/SPIClassIterator.java b/lucene/core/src/java/org/apache/lucene/util/SPIClassIterator.java
index 764713c88a6..5ea20c4aaeb 100644
--- a/lucene/core/src/java/org/apache/lucene/util/SPIClassIterator.java
+++ b/lucene/core/src/java/org/apache/lucene/util/SPIClassIterator.java
@@ -22,6 +22,7 @@ import java.io.InputStream;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Enumeration;
@@ -91,7 +92,7 @@ public final class SPIClassIterator implements Iterator> {
final InputStream in = url.openStream();
IOException priorE = null;
try {
- final BufferedReader reader = new BufferedReader(new InputStreamReader(in, IOUtils.CHARSET_UTF_8));
+ final BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
String line;
while ((line = reader.readLine()) != null) {
final int pos = line.indexOf('#');
diff --git a/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java b/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java
index 70f5408781a..13f1b10f9ba 100644
--- a/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java
@@ -1,5 +1,7 @@
package org.apache.lucene.util;
+import java.nio.charset.StandardCharsets;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -88,7 +90,7 @@ package org.apache.lucene.util;
/**
* Class to encode java's UTF16 char[] into UTF8 byte[]
* without always allocating a new byte[] as
- * String.getBytes("UTF-8") does.
+ * String.getBytes(StandardCharsets.UTF_8) does.
*
* @lucene.internal
*/
@@ -292,7 +294,7 @@ public final class UnicodeUtil {
private static boolean matches(char[] source, int offset, int length, byte[] result, int upto) {
try {
String s1 = new String(source, offset, length);
- String s2 = new String(result, 0, upto, "UTF-8");
+ String s2 = new String(result, 0, upto, StandardCharsets.UTF_8);
if (!s1.equals(s2)) {
//System.out.println("DIFF: s1 len=" + s1.length());
//for(int i=0;i {
/*
if (bytes.length == 665) {
- Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"), "UTF-8");
+ Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"), StandardCharsets.UTF_8);
Util.toDot(this, w, false, false);
w.close();
System.out.println("Wrote FST to out.dot");
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestLZ4CompressionMode.java b/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestLZ4CompressionMode.java
index c7978fdb92b..1cfdb446227 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestLZ4CompressionMode.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestLZ4CompressionMode.java
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.compressing;
*/
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import com.carrotsearch.randomizedtesting.generators.RandomInts;
@@ -81,7 +82,7 @@ public abstract class AbstractTestLZ4CompressionMode extends AbstractTestCompres
public void testShortLiteralsAndMatchs() throws IOException {
// literals and matchs lengths <= 15
- final byte[] decompressed = "1234562345673456745678910123".getBytes("UTF-8");
+ final byte[] decompressed = "1234562345673456745678910123".getBytes(StandardCharsets.UTF_8);
test(decompressed);
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java
index 050770a486f..ada8380dcc6 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java
@@ -23,7 +23,7 @@ import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
import org.apache.lucene.codecs.lucene46.Lucene46Codec;
-import org.apache.lucene.codecs.mocksep.MockSepPostingsFormat;
+import org.apache.lucene.codecs.memory.MemoryPostingsFormat;
import org.apache.lucene.codecs.pulsing.Pulsing41PostingsFormat;
import org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat;
import org.apache.lucene.document.Document;
@@ -44,7 +44,6 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
-import org.apache.lucene.util.TestUtil;
import org.junit.Test;
/**
@@ -204,14 +203,14 @@ public class TestPerFieldPostingsFormat2 extends LuceneTestCase {
public static class MockCodec extends Lucene46Codec {
final PostingsFormat lucene40 = new Lucene41PostingsFormat();
final PostingsFormat simpleText = new SimpleTextPostingsFormat();
- final PostingsFormat mockSep = new MockSepPostingsFormat();
+ final PostingsFormat memory = new MemoryPostingsFormat();
@Override
public PostingsFormat getPostingsFormatForField(String field) {
if (field.equals("id")) {
return simpleText;
} else if (field.equals("content")) {
- return mockSep;
+ return memory;
} else {
return lucene40;
}
diff --git a/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java b/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java
index c001f82306b..5819adcc1ca 100644
--- a/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java
+++ b/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java
@@ -1,5 +1,7 @@
package org.apache.lucene.document;
+import java.nio.charset.StandardCharsets;
+
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.RandomIndexWriter;
@@ -38,7 +40,7 @@ public class TestBinaryDocument extends LuceneTestCase {
{
FieldType ft = new FieldType();
ft.setStored(true);
- StoredField binaryFldStored = new StoredField("binaryStored", binaryValStored.getBytes("UTF-8"));
+ StoredField binaryFldStored = new StoredField("binaryStored", binaryValStored.getBytes(StandardCharsets.UTF_8));
Field stringFldStored = new Field("stringStored", binaryValStored, ft);
Document doc = new Document();
@@ -63,7 +65,7 @@ public class TestBinaryDocument extends LuceneTestCase {
/** fetch the binary stored field and compare it's content with the original one */
BytesRef bytes = docFromReader.getBinaryValue("binaryStored");
assertNotNull(bytes);
- String binaryFldStoredTest = new String(bytes.bytes, bytes.offset, bytes.length, "UTF-8");
+ String binaryFldStoredTest = new String(bytes.bytes, bytes.offset, bytes.length, StandardCharsets.UTF_8);
assertTrue(binaryFldStoredTest.equals(binaryValStored));
/** fetch the string field and compare it's content with the original one */
@@ -76,7 +78,7 @@ public class TestBinaryDocument extends LuceneTestCase {
}
public void testCompressionTools() throws Exception {
- StoredField binaryFldCompressed = new StoredField("binaryCompressed", CompressionTools.compress(binaryValCompressed.getBytes("UTF-8")));
+ StoredField binaryFldCompressed = new StoredField("binaryCompressed", CompressionTools.compress(binaryValCompressed.getBytes(StandardCharsets.UTF_8)));
StoredField stringFldCompressed = new StoredField("stringCompressed", CompressionTools.compressString(binaryValCompressed));
Document doc = new Document();
@@ -95,7 +97,7 @@ public class TestBinaryDocument extends LuceneTestCase {
assertTrue(docFromReader != null);
/** fetch the binary compressed field and compare it's content with the original one */
- String binaryFldCompressedTest = new String(CompressionTools.decompress(docFromReader.getBinaryValue("binaryCompressed")), "UTF-8");
+ String binaryFldCompressedTest = new String(CompressionTools.decompress(docFromReader.getBinaryValue("binaryCompressed")), StandardCharsets.UTF_8);
assertTrue(binaryFldCompressedTest.equals(binaryValCompressed));
assertTrue(CompressionTools.decompressString(docFromReader.getBinaryValue("stringCompressed")).equals(binaryValCompressed));
diff --git a/lucene/core/src/test/org/apache/lucene/document/TestDocument.java b/lucene/core/src/test/org/apache/lucene/document/TestDocument.java
index e9c9e5be9a7..2d5d7592a94 100644
--- a/lucene/core/src/test/org/apache/lucene/document/TestDocument.java
+++ b/lucene/core/src/test/org/apache/lucene/document/TestDocument.java
@@ -19,6 +19,7 @@ package org.apache.lucene.document;
import java.io.IOException;
import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import org.apache.lucene.analysis.MockTokenizer;
@@ -53,8 +54,8 @@ public class TestDocument extends LuceneTestCase {
FieldType ft = new FieldType();
ft.setStored(true);
Field stringFld = new Field("string", binaryVal, ft);
- StoredField binaryFld = new StoredField("binary", binaryVal.getBytes("UTF-8"));
- StoredField binaryFld2 = new StoredField("binary", binaryVal2.getBytes("UTF-8"));
+ StoredField binaryFld = new StoredField("binary", binaryVal.getBytes(StandardCharsets.UTF_8));
+ StoredField binaryFld2 = new StoredField("binary", binaryVal2.getBytes(StandardCharsets.UTF_8));
doc.add(stringFld);
doc.add(binaryFld);
diff --git a/lucene/core/src/test/org/apache/lucene/document/TestField.java b/lucene/core/src/test/org/apache/lucene/document/TestField.java
index 2417c65069b..62be66aa8ce 100644
--- a/lucene/core/src/test/org/apache/lucene/document/TestField.java
+++ b/lucene/core/src/test/org/apache/lucene/document/TestField.java
@@ -18,6 +18,8 @@ package org.apache.lucene.document;
*/
import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+
import org.apache.lucene.analysis.CannedTokenStream;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.util.BytesRef;
@@ -184,7 +186,7 @@ public class TestField extends LuceneTestCase {
trySetBoost(field);
trySetByteValue(field);
- field.setBytesValue("fubar".getBytes("UTF-8"));
+ field.setBytesValue("fubar".getBytes(StandardCharsets.UTF_8));
field.setBytesValue(new BytesRef("baz"));
trySetDoubleValue(field);
trySetIntValue(field);
@@ -203,7 +205,7 @@ public class TestField extends LuceneTestCase {
trySetBoost(field);
trySetByteValue(field);
- field.setBytesValue("fubar".getBytes("UTF-8"));
+ field.setBytesValue("fubar".getBytes(StandardCharsets.UTF_8));
field.setBytesValue(new BytesRef("baz"));
trySetDoubleValue(field);
trySetIntValue(field);
@@ -294,15 +296,15 @@ public class TestField extends LuceneTestCase {
public void testStoredFieldBytes() throws Exception {
Field fields[] = new Field[] {
- new StoredField("foo", "bar".getBytes("UTF-8")),
- new StoredField("foo", "bar".getBytes("UTF-8"), 0, 3),
+ new StoredField("foo", "bar".getBytes(StandardCharsets.UTF_8)),
+ new StoredField("foo", "bar".getBytes(StandardCharsets.UTF_8), 0, 3),
new StoredField("foo", new BytesRef("bar")),
};
for (Field field : fields) {
trySetBoost(field);
trySetByteValue(field);
- field.setBytesValue("baz".getBytes("UTF-8"));
+ field.setBytesValue("baz".getBytes(StandardCharsets.UTF_8));
field.setBytesValue(new BytesRef("baz"));
trySetDoubleValue(field);
trySetIntValue(field);
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java b/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
index 27fd97ef298..50850fac27f 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
@@ -1064,14 +1064,14 @@ public class TestAddIndexes extends LuceneTestCase {
private static final class CustomPerFieldCodec extends Lucene46Codec {
private final PostingsFormat simpleTextFormat = PostingsFormat.forName("SimpleText");
private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene41");
- private final PostingsFormat mockSepFormat = PostingsFormat.forName("MockSep");
+ private final PostingsFormat memoryFormat = PostingsFormat.forName("Memory");
@Override
public PostingsFormat getPostingsFormatForField(String field) {
if (field.equals("id")) {
return simpleTextFormat;
} else if (field.equals("content")) {
- return mockSepFormat;
+ return memoryFormat;
} else {
return defaultFormat;
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java
new file mode 100644
index 00000000000..7f7e9bf1197
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java
@@ -0,0 +1,90 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.lucene46.Lucene46Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.store.CompoundFileDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
+
+/**
+ * Test that a plain default puts CRC32 footers in all files.
+ */
+public class TestAllFilesHaveChecksumFooter extends LuceneTestCase {
+ public void test() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ conf.setCodec(new Lucene46Codec());
+ RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
+ Document doc = new Document();
+ // these fields should sometimes get term vectors, etc
+ Field idField = newStringField("id", "", Field.Store.NO);
+ Field bodyField = newTextField("body", "", Field.Store.NO);
+ Field dvField = new NumericDocValuesField("dv", 5);
+ doc.add(idField);
+ doc.add(bodyField);
+ doc.add(dvField);
+ for (int i = 0; i < 100; i++) {
+ idField.setStringValue(Integer.toString(i));
+ bodyField.setStringValue(TestUtil.randomUnicodeString(random()));
+ riw.addDocument(doc);
+ if (random().nextInt(7) == 0) {
+ riw.commit();
+ }
+ if (random().nextInt(20) == 0) {
+ riw.deleteDocuments(new Term("id", Integer.toString(i)));
+ }
+ }
+ riw.close();
+ checkHeaders(dir);
+ dir.close();
+ }
+
+ private void checkHeaders(Directory dir) throws IOException {
+ for (String file : dir.listAll()) {
+ if (file.endsWith(IndexFileNames.COMPOUND_FILE_EXTENSION)) {
+ CompoundFileDirectory cfsDir = new CompoundFileDirectory(dir, file, newIOContext(random()), false);
+ checkHeaders(cfsDir); // recurse into cfs
+ cfsDir.close();
+ }
+ IndexInput in = null;
+ boolean success = false;
+ try {
+ in = dir.openInput(file, newIOContext(random()));
+ CodecUtil.checksumEntireFile(in);
+ success = true;
+ } finally {
+ if (success) {
+ IOUtils.close(in);
+ } else {
+ IOUtils.closeWhileHandlingException(in);
+ }
+ }
+ }
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
index defea12a415..e4c998eb861 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
@@ -24,6 +24,7 @@ import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.lucene46.Lucene46Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
@@ -39,14 +40,15 @@ public class TestAllFilesHaveCodecHeader extends LuceneTestCase {
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
conf.setCodec(new Lucene46Codec());
- // riw should sometimes create docvalues fields, etc
RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
Document doc = new Document();
// these fields should sometimes get term vectors, etc
Field idField = newStringField("id", "", Field.Store.NO);
Field bodyField = newTextField("body", "", Field.Store.NO);
+ Field dvField = new NumericDocValuesField("dv", 5);
doc.add(idField);
doc.add(bodyField);
+ doc.add(dvField);
for (int i = 0; i < 100; i++) {
idField.setStringValue(Integer.toString(i));
bodyField.setStringValue(TestUtil.randomUnicodeString(random()));
@@ -54,6 +56,10 @@ public class TestAllFilesHaveCodecHeader extends LuceneTestCase {
if (random().nextInt(7) == 0) {
riw.commit();
}
+ // TODO: we should make a new format with a clean header...
+ // if (random().nextInt(20) == 0) {
+ // riw.deleteDocuments(new Term("id", Integer.toString(i)));
+ // }
}
riw.close();
checkHeaders(dir);
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index 3ba113c417e..4c82757e2c2 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -60,6 +60,7 @@ import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.StringHelper;
@@ -76,7 +77,7 @@ import org.junit.BeforeClass;
// we won't even be running the actual code, only the impostor
// @SuppressCodecs("Lucene4x")
// Sep codec cannot yet handle the offsets in our 4.x index!
-@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom", "Lucene40", "Lucene41", "Lucene42", "Lucene45"})
+@SuppressCodecs({"Lucene40", "Lucene41", "Lucene42", "Lucene45"})
public class TestBackwardsCompatibility extends LuceneTestCase {
// Uncomment these cases & run them on an older Lucene version,
@@ -292,13 +293,13 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
CheckIndex checker = new CheckIndex(dir);
- checker.setInfoStream(new PrintStream(bos, false, "UTF-8"));
+ checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8));
CheckIndex.Status indexStatus = checker.checkIndex();
assertFalse(indexStatus.clean);
- assertTrue(bos.toString("UTF-8").contains(IndexFormatTooOldException.class.getName()));
+ assertTrue(bos.toString(IOUtils.UTF_8).contains(IndexFormatTooOldException.class.getName()));
dir.close();
- TestUtil.rmDir(oldIndxeDir);
+ TestUtil.rm(oldIndxeDir);
}
}
@@ -597,7 +598,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
public File createIndex(String dirName, boolean doCFS, boolean fullyMerged) throws IOException {
// we use a real directory name that is not cleaned up, because this method is only used to create backwards indexes:
File indexDir = new File("/tmp/idx", dirName);
- TestUtil.rmDir(indexDir);
+ TestUtil.rm(indexDir);
Directory dir = newFSDirectory(indexDir);
LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();
mp.setNoCFSRatio(doCFS ? 1.0 : 0.0);
@@ -646,7 +647,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
String outputDirName = "lucene.backwardscompat0.index";
File outputDir = TestUtil.createTempDir(outputDirName);
- TestUtil.rmDir(outputDir);
+ TestUtil.rm(outputDir);
try {
Directory dir = newFSDirectory(outputDir);
@@ -704,7 +705,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
}
dir.close();
} finally {
- TestUtil.rmDir(outputDir);
+ TestUtil.rm(outputDir);
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestBagOfPositions.java b/lucene/core/src/test/org/apache/lucene/index/TestBagOfPositions.java
index 4df23640c64..70bdfa54794 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestBagOfPositions.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestBagOfPositions.java
@@ -87,7 +87,7 @@ public class TestBagOfPositions extends LuceneTestCase {
if (options == 0) {
fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS); // we dont actually need positions
fieldType.setStoreTermVectors(true); // but enforce term vectors when we do this so we check SOMETHING
- } else if (options == 1 && !doesntSupportOffsets.contains(TestUtil.getPostingsFormat("field"))) {
+ } else if (options == 1) {
fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
}
// else just positions
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java b/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java
index 9d13b1aebbe..67b719e7fea 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java
@@ -20,9 +20,11 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.ArrayList;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.store.Directory;
import org.apache.lucene.analysis.CannedTokenStream;
@@ -54,12 +56,12 @@ public class TestCheckIndex extends LuceneTestCase {
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
CheckIndex checker = new CheckIndex(dir);
- checker.setInfoStream(new PrintStream(bos, false, "UTF-8"));
+ checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8));
if (VERBOSE) checker.setInfoStream(System.out);
CheckIndex.Status indexStatus = checker.checkIndex();
if (indexStatus.clean == false) {
System.out.println("CheckIndex failed");
- System.out.println(bos.toString("UTF-8"));
+ System.out.println(bos.toString(IOUtils.UTF_8));
fail();
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
index 5b33629d3b7..21f1c6d0f33 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
@@ -29,7 +29,6 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.lucene40.Lucene40RWCodec;
import org.apache.lucene.codecs.lucene41.Lucene41RWCodec;
import org.apache.lucene.codecs.lucene42.Lucene42RWCodec;
-import org.apache.lucene.codecs.mocksep.MockSepPostingsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.FieldType;
@@ -336,54 +335,6 @@ public class TestCodecs extends LuceneTestCase {
dir.close();
}
- public void testSepPositionAfterMerge() throws IOException {
- final Directory dir = newDirectory();
- final IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
- new MockAnalyzer(random()));
- config.setMergePolicy(newLogMergePolicy());
- config.setCodec(TestUtil.alwaysPostingsFormat(new MockSepPostingsFormat()));
- final IndexWriter writer = new IndexWriter(dir, config);
-
- try {
- final PhraseQuery pq = new PhraseQuery();
- pq.add(new Term("content", "bbb"));
- pq.add(new Term("content", "ccc"));
-
- final Document doc = new Document();
- FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
- customType.setOmitNorms(true);
- doc.add(newField("content", "aaa bbb ccc ddd", customType));
-
- // add document and force commit for creating a first segment
- writer.addDocument(doc);
- writer.commit();
-
- ScoreDoc[] results = this.search(writer, pq, 5);
- assertEquals(1, results.length);
- assertEquals(0, results[0].doc);
-
- // add document and force commit for creating a second segment
- writer.addDocument(doc);
- writer.commit();
-
- // at this point, there should be at least two segments
- results = this.search(writer, pq, 5);
- assertEquals(2, results.length);
- assertEquals(0, results[0].doc);
-
- writer.forceMerge(1);
-
- // optimise to merge the segments.
- results = this.search(writer, pq, 5);
- assertEquals(2, results.length);
- assertEquals(0, results[0].doc);
- }
- finally {
- writer.close();
- dir.close();
- }
- }
-
private ScoreDoc[] search(final IndexWriter writer, final Query q, final int n) throws IOException {
final IndexReader reader = writer.getReader();
final IndexSearcher searcher = newSearcher(reader);
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
index 2a786f42cc8..80956f99dbe 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
@@ -25,6 +25,7 @@ import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
@@ -89,7 +90,7 @@ public class TestDoc extends LuceneTestCase {
File f = new File(workDir, name);
if (f.exists()) f.delete();
- fw = new OutputStreamWriter(new FileOutputStream(f), "UTF-8");
+ fw = new OutputStreamWriter(new FileOutputStream(f), StandardCharsets.UTF_8);
pw = new PrintWriter(fw);
pw.println(text);
return f;
@@ -200,7 +201,7 @@ public class TestDoc extends LuceneTestCase {
{
File file = new File(workDir, fileName);
Document doc = new Document();
- InputStreamReader is = new InputStreamReader(new FileInputStream(file), "UTF-8");
+ InputStreamReader is = new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8);
doc.add(new TextField("contents", is));
writer.addDocument(doc);
writer.commit();
@@ -221,7 +222,7 @@ public class TestDoc extends LuceneTestCase {
SegmentMerger merger = new SegmentMerger(Arrays.asList(r1, r2),
si, InfoStream.getDefault(), trackingDir,
- MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), context);
+ MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), context, true);
MergeState mergeState = merger.merge();
r1.close();
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocInverterPerFieldErrorInfo.java b/lucene/core/src/test/org/apache/lucene/index/TestDocInverterPerFieldErrorInfo.java
index 3df1ebc7db4..c2963c97e68 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDocInverterPerFieldErrorInfo.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDocInverterPerFieldErrorInfo.java
@@ -26,6 +26,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.PrintStreamInfoStream;
import org.junit.Test;
@@ -70,7 +71,7 @@ public class TestDocInverterPerFieldErrorInfo extends LuceneTestCase {
IndexWriter writer;
IndexWriterConfig c = new IndexWriterConfig(TEST_VERSION_CURRENT, new ThrowingAnalyzer());
final ByteArrayOutputStream infoBytes = new ByteArrayOutputStream();
- PrintStream infoPrintStream = new PrintStream(infoBytes, true, "utf-8");
+ PrintStream infoPrintStream = new PrintStream(infoBytes, true, IOUtils.UTF_8);
PrintStreamInfoStream printStreamInfoStream = new PrintStreamInfoStream(infoPrintStream);
c.setInfoStream(printStreamInfoStream);
writer = new IndexWriter(dir, c);
@@ -81,7 +82,7 @@ public class TestDocInverterPerFieldErrorInfo extends LuceneTestCase {
fail("Failed to fail.");
} catch(BadNews badNews) {
infoPrintStream.flush();
- String infoStream = new String(infoBytes.toByteArray(), "utf-8");
+ String infoStream = new String(infoBytes.toByteArray(), IOUtils.UTF_8);
assertTrue(infoStream.contains("distinctiveFieldName"));
}
@@ -95,7 +96,7 @@ public class TestDocInverterPerFieldErrorInfo extends LuceneTestCase {
IndexWriter writer;
IndexWriterConfig c = new IndexWriterConfig(TEST_VERSION_CURRENT, new ThrowingAnalyzer());
final ByteArrayOutputStream infoBytes = new ByteArrayOutputStream();
- PrintStream infoPrintStream = new PrintStream(infoBytes, true, "utf-8");
+ PrintStream infoPrintStream = new PrintStream(infoBytes, true, IOUtils.UTF_8);
PrintStreamInfoStream printStreamInfoStream = new PrintStreamInfoStream(infoPrintStream);
c.setInfoStream(printStreamInfoStream);
writer = new IndexWriter(dir, c);
@@ -107,7 +108,7 @@ public class TestDocInverterPerFieldErrorInfo extends LuceneTestCase {
fail("Unwanted exception");
}
infoPrintStream.flush();
- String infoStream = new String(infoBytes.toByteArray(), "utf-8");
+ String infoStream = new String(infoBytes.toByteArray(), IOUtils.UTF_8);
assertFalse(infoStream.contains("boringFieldName"));
writer.close();
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
index caec565055b..f5e3dbf3e3f 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -43,6 +44,7 @@ import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -1173,10 +1175,10 @@ public class TestIndexWriterDelete extends LuceneTestCase {
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
CheckIndex checker = new CheckIndex(dir);
- checker.setInfoStream(new PrintStream(bos, false, "UTF-8"), false);
+ checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false);
CheckIndex.Status indexStatus = checker.checkIndex(null);
assertTrue(indexStatus.clean);
- String s = bos.toString("UTF-8");
+ String s = bos.toString(IOUtils.UTF_8);
// Segment should have deletions:
assertTrue(s.contains("has deletions"));
@@ -1185,10 +1187,10 @@ public class TestIndexWriterDelete extends LuceneTestCase {
w.close();
bos = new ByteArrayOutputStream(1024);
- checker.setInfoStream(new PrintStream(bos, false, "UTF-8"), false);
+ checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false);
indexStatus = checker.checkIndex(null);
assertTrue(indexStatus.clean);
- s = bos.toString("UTF-8");
+ s = bos.toString(IOUtils.UTF_8);
assertFalse(s.contains("has deletions"));
dir.close();
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterUnicode.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterUnicode.java
index be3170d8660..1c202cfa7bd 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterUnicode.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterUnicode.java
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Random;
@@ -181,7 +182,7 @@ public class TestIndexWriterUnicode extends LuceneTestCase {
UnicodeUtil.UTF16toUTF8(buffer, 0, 20, utf8);
if (!hasIllegal) {
- byte[] b = new String(buffer, 0, 20).getBytes("UTF-8");
+ byte[] b = new String(buffer, 0, 20).getBytes(StandardCharsets.UTF_8);
assertEquals(b.length, utf8.length);
for(int i=0;iasList(reader1, reader2),
si, InfoStream.getDefault(), mergedDir,
- MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), newIOContext(random()));
+ MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), newIOContext(random()), true);
MergeState mergeState = merger.merge();
int docsMerged = mergeState.segmentInfo.getDocCount();
assertTrue(docsMerged == 2);
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java b/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
index f78f6f84bf9..e490518412e 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
@@ -60,6 +60,7 @@ import org.apache.lucene.search.FieldCache.Longs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.TestUtil;
@@ -139,7 +140,7 @@ public class TestFieldCache extends LuceneTestCase {
try {
FieldCache cache = FieldCache.DEFAULT;
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
- cache.setInfoStream(new PrintStream(bos, false, "UTF-8"));
+ cache.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8));
cache.getDoubles(reader, "theDouble", false);
cache.getFloats(reader, "theDouble", new FieldCache.FloatParser() {
@Override
@@ -151,7 +152,7 @@ public class TestFieldCache extends LuceneTestCase {
return NumericUtils.sortableIntToFloat((int) NumericUtils.prefixCodedToLong(term));
}
}, false);
- assertTrue(bos.toString("UTF-8").indexOf("WARNING") != -1);
+ assertTrue(bos.toString(IOUtils.UTF_8).indexOf("WARNING") != -1);
} finally {
FieldCache.DEFAULT.purgeAllCaches();
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java b/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java
index 123b256ba70..1e918d56d3c 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java
@@ -20,6 +20,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
import java.util.Collection;
import org.apache.lucene.analysis.*;
@@ -249,7 +250,7 @@ public class TestPositionIncrement extends LuceneTestCase {
for (byte[] bytes : payloads) {
count++;
if (VERBOSE) {
- System.out.println(" payload: " + new String(bytes, "UTF-8"));
+ System.out.println(" payload: " + new String(bytes, StandardCharsets.UTF_8));
}
}
}
@@ -276,7 +277,7 @@ public class TestPositionIncrement extends LuceneTestCase {
Collection pls = psu.getPayloadsForQuery(snq);
count = pls.size();
for (byte[] bytes : pls) {
- String s = new String(bytes, "UTF-8");
+ String s = new String(bytes, StandardCharsets.UTF_8);
//System.out.println(s);
sawZero |= s.equals("pos: 0");
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java
index 53314e84b42..d5ca61b430e 100644
--- a/lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java
+++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java
@@ -18,6 +18,7 @@ package org.apache.lucene.search.spans;
*/
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -80,7 +81,7 @@ public class TestBasics extends LuceneTestCase {
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
- payloadAttr.setPayload(new BytesRef(("pos: " + pos).getBytes("UTF-8")));
+ payloadAttr.setPayload(new BytesRef(("pos: " + pos).getBytes(StandardCharsets.UTF_8)));
pos++;
return true;
} else {
@@ -482,7 +483,7 @@ public class TestBasics extends LuceneTestCase {
@Test
public void testSpanPayloadCheck() throws Exception {
SpanTermQuery term1 = new SpanTermQuery(new Term("field", "five"));
- BytesRef pay = new BytesRef(("pos: " + 5).getBytes("UTF-8"));
+ BytesRef pay = new BytesRef(("pos: " + 5).getBytes(StandardCharsets.UTF_8));
SpanQuery query = new SpanPayloadCheckQuery(term1, Collections.singletonList(pay.bytes));
checkHits(query, new int[]
{1125, 1135, 1145, 1155, 1165, 1175, 1185, 1195, 1225, 1235, 1245, 1255, 1265, 1275, 1285, 1295, 1325, 1335, 1345, 1355, 1365, 1375, 1385, 1395, 1425, 1435, 1445, 1455, 1465, 1475, 1485, 1495, 1525, 1535, 1545, 1555, 1565, 1575, 1585, 1595, 1625, 1635, 1645, 1655, 1665, 1675, 1685, 1695, 1725, 1735, 1745, 1755, 1765, 1775, 1785, 1795, 1825, 1835, 1845, 1855, 1865, 1875, 1885, 1895, 1925, 1935, 1945, 1955, 1965, 1975, 1985, 1995});
@@ -497,8 +498,8 @@ public class TestBasics extends LuceneTestCase {
clauses[0] = term1;
clauses[1] = term2;
snq = new SpanNearQuery(clauses, 0, true);
- pay = new BytesRef(("pos: " + 0).getBytes("UTF-8"));
- pay2 = new BytesRef(("pos: " + 1).getBytes("UTF-8"));
+ pay = new BytesRef(("pos: " + 0).getBytes(StandardCharsets.UTF_8));
+ pay2 = new BytesRef(("pos: " + 1).getBytes(StandardCharsets.UTF_8));
list = new ArrayList<>();
list.add(pay.bytes);
list.add(pay2.bytes);
@@ -510,9 +511,9 @@ public class TestBasics extends LuceneTestCase {
clauses[1] = term2;
clauses[2] = new SpanTermQuery(new Term("field", "five"));
snq = new SpanNearQuery(clauses, 0, true);
- pay = new BytesRef(("pos: " + 0).getBytes("UTF-8"));
- pay2 = new BytesRef(("pos: " + 1).getBytes("UTF-8"));
- BytesRef pay3 = new BytesRef(("pos: " + 2).getBytes("UTF-8"));
+ pay = new BytesRef(("pos: " + 0).getBytes(StandardCharsets.UTF_8));
+ pay2 = new BytesRef(("pos: " + 1).getBytes(StandardCharsets.UTF_8));
+ BytesRef pay3 = new BytesRef(("pos: " + 2).getBytes(StandardCharsets.UTF_8));
list = new ArrayList<>();
list.add(pay.bytes);
list.add(pay2.bytes);
@@ -541,10 +542,10 @@ public class TestBasics extends LuceneTestCase {
checkHits(query, new int[]{1103, 1203,1303,1403,1503,1603,1703,1803,1903});
Collection payloads = new ArrayList<>();
- BytesRef pay = new BytesRef(("pos: " + 0).getBytes("UTF-8"));
- BytesRef pay2 = new BytesRef(("pos: " + 1).getBytes("UTF-8"));
- BytesRef pay3 = new BytesRef(("pos: " + 3).getBytes("UTF-8"));
- BytesRef pay4 = new BytesRef(("pos: " + 4).getBytes("UTF-8"));
+ BytesRef pay = new BytesRef(("pos: " + 0).getBytes(StandardCharsets.UTF_8));
+ BytesRef pay2 = new BytesRef(("pos: " + 1).getBytes(StandardCharsets.UTF_8));
+ BytesRef pay3 = new BytesRef(("pos: " + 3).getBytes(StandardCharsets.UTF_8));
+ BytesRef pay4 = new BytesRef(("pos: " + 4).getBytes(StandardCharsets.UTF_8));
payloads.add(pay.bytes);
payloads.add(pay2.bytes);
payloads.add(pay3.bytes);
diff --git a/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java b/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
index 485430b5ed7..ecd16c7d132 100644
--- a/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
+++ b/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
@@ -19,6 +19,7 @@ package org.apache.lucene.search.spans;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
@@ -276,7 +277,7 @@ public class TestPayloadSpans extends LuceneTestCase {
Collection payloads = spans.getPayload();
for (final byte [] payload : payloads) {
- payloadSet.add(new String(payload, "UTF-8"));
+ payloadSet.add(new String(payload, StandardCharsets.UTF_8));
}
}
}
@@ -311,7 +312,7 @@ public class TestPayloadSpans extends LuceneTestCase {
while (spans.next()) {
Collection payloads = spans.getPayload();
for (final byte[] payload : payloads) {
- payloadSet.add(new String(payload, "UTF-8"));
+ payloadSet.add(new String(payload, StandardCharsets.UTF_8));
}
}
}
@@ -347,7 +348,7 @@ public class TestPayloadSpans extends LuceneTestCase {
Collection payloads = spans.getPayload();
for (final byte [] payload : payloads) {
- payloadSet.add(new String(payload, "UTF-8"));
+ payloadSet.add(new String(payload, StandardCharsets.UTF_8));
}
}
}
@@ -382,7 +383,7 @@ public class TestPayloadSpans extends LuceneTestCase {
if(VERBOSE) {
System.out.println("Num payloads:" + payloads.size());
for (final byte [] bytes : payloads) {
- System.out.println(new String(bytes, "UTF-8"));
+ System.out.println(new String(bytes, StandardCharsets.UTF_8));
}
}
reader.close();
@@ -455,7 +456,7 @@ public class TestPayloadSpans extends LuceneTestCase {
System.out.println("payloads for span:" + payload.size());
for (final byte [] bytes : payload) {
System.out.println("doc:" + spans.doc() + " s:" + spans.start() + " e:" + spans.end() + " "
- + new String(bytes, "UTF-8"));
+ + new String(bytes, StandardCharsets.UTF_8));
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestBufferedChecksum.java b/lucene/core/src/test/org/apache/lucene/store/TestBufferedChecksum.java
new file mode 100644
index 00000000000..c7915d40b96
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/store/TestBufferedChecksum.java
@@ -0,0 +1,68 @@
+package org.apache.lucene.store;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.zip.CRC32;
+import java.util.zip.Checksum;
+
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestBufferedChecksum extends LuceneTestCase {
+
+ public void testSimple() {
+ Checksum c = new BufferedChecksum(new CRC32());
+ c.update(1);
+ c.update(2);
+ c.update(3);
+ assertEquals(1438416925L, c.getValue());
+ }
+
+ public void testRandom() {
+ Checksum c1 = new CRC32();
+ Checksum c2 = new BufferedChecksum(new CRC32());
+ int iterations = atLeast(10000);
+ for (int i = 0; i < iterations; i++) {
+ switch(random().nextInt(4)) {
+ case 0:
+ // update(byte[], int, int)
+ int length = random().nextInt(1024);
+ byte bytes[] = new byte[length];
+ random().nextBytes(bytes);
+ c1.update(bytes, 0, bytes.length);
+ c2.update(bytes, 0, bytes.length);
+ break;
+ case 1:
+ // update(int)
+ int b = random().nextInt(256);
+ c1.update(b);
+ c2.update(b);
+ break;
+ case 2:
+ // reset()
+ c1.reset();
+ c2.reset();
+ break;
+ case 3:
+ // getValue()
+ assertEquals(c1.getValue(), c2.getValue());
+ break;
+ }
+ }
+ assertEquals(c1.getValue(), c2.getValue());
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java
index 577315be3df..42941d419b2 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java
@@ -30,12 +30,13 @@ public class TestFilterDirectory extends LuceneTestCase {
public void testOverrides() throws Exception {
// verify that all methods of Directory are overridden by FilterDirectory,
// except those under the 'exclude' list
- Set exclude = new HashSet<>();
- exclude.add("copy");
- exclude.add("createSlicer");
+ Set exclude = new HashSet<>();
+ exclude.add(Directory.class.getMethod("copy", Directory.class, String.class, String.class, IOContext.class));
+ exclude.add(Directory.class.getMethod("createSlicer", String.class, IOContext.class));
+ exclude.add(Directory.class.getMethod("openChecksumInput", String.class, IOContext.class));
for (Method m : FilterDirectory.class.getMethods()) {
if (m.getDeclaringClass() == Directory.class) {
- assertTrue("method " + m.getName() + " not overridden!", exclude.contains(m.getName()));
+ assertTrue("method " + m.getName() + " not overridden!", exclude.contains(m));
}
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestMaxFailuresRule.java b/lucene/core/src/test/org/apache/lucene/util/TestMaxFailuresRule.java
index 8dd4144de98..d8154169d2d 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestMaxFailuresRule.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestMaxFailuresRule.java
@@ -17,8 +17,11 @@ package org.apache.lucene.util;
* limitations under the License.
*/
+import java.util.concurrent.CountDownLatch;
+
import org.apache.lucene.util.junitcompat.WithNestedTests;
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.Description;
@@ -27,8 +30,13 @@ import org.junit.runner.Result;
import org.junit.runner.notification.Failure;
import org.junit.runner.notification.RunListener;
-import com.carrotsearch.randomizedtesting.SysGlobals;
import com.carrotsearch.randomizedtesting.annotations.Repeat;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies.Consequence;
import com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule;
import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;
@@ -66,48 +74,118 @@ public class TestMaxFailuresRule extends WithNestedTests {
@Test
public void testMaxFailures() {
- TestRuleIgnoreAfterMaxFailures newRule = new TestRuleIgnoreAfterMaxFailures(2);
- TestRuleIgnoreAfterMaxFailures prevRule = LuceneTestCase.replaceMaxFailureRule(newRule);
- System.clearProperty(SysGlobals.SYSPROP_ITERATIONS());
- try {
- JUnitCore core = new JUnitCore();
- final StringBuilder results = new StringBuilder();
- core.addListener(new RunListener() {
- char lastTest;
+ LuceneTestCase.replaceMaxFailureRule(new TestRuleIgnoreAfterMaxFailures(2));
+ JUnitCore core = new JUnitCore();
+ final StringBuilder results = new StringBuilder();
+ core.addListener(new RunListener() {
+ char lastTest;
- @Override
- public void testStarted(Description description) throws Exception {
- lastTest = 'S'; // success.
- }
+ @Override
+ public void testStarted(Description description) throws Exception {
+ lastTest = 'S'; // success.
+ }
- @Override
- public void testAssumptionFailure(Failure failure) {
- lastTest = 'A'; // assumption failure.
- }
+ @Override
+ public void testAssumptionFailure(Failure failure) {
+ lastTest = 'A'; // assumption failure.
+ }
- @Override
- public void testFailure(Failure failure) throws Exception {
- lastTest = 'F'; // failure
- }
+ @Override
+ public void testFailure(Failure failure) throws Exception {
+ lastTest = 'F'; // failure
+ }
- @Override
- public void testFinished(Description description) throws Exception {
- results.append(lastTest);
- }
- });
+ @Override
+ public void testFinished(Description description) throws Exception {
+ results.append(lastTest);
+ }
+ });
- Result result = core.run(Nested.class);
- Assert.assertEquals(500, result.getRunCount());
- Assert.assertEquals(0, result.getIgnoreCount());
- Assert.assertEquals(2, result.getFailureCount());
+ Result result = core.run(Nested.class);
+ Assert.assertEquals(500, result.getRunCount());
+ Assert.assertEquals(0, result.getIgnoreCount());
+ Assert.assertEquals(2, result.getFailureCount());
- // Make sure we had exactly two failures followed by assumption-failures
- // resulting from ignored tests.
- Assert.assertTrue(results.toString(),
- results.toString().matches("(S*F){2}A+"));
+ // Make sure we had exactly two failures followed by assumption-failures
+ // resulting from ignored tests.
+ Assert.assertTrue(results.toString(),
+ results.toString().matches("(S*F){2}A+"));
+ }
- } finally {
- LuceneTestCase.replaceMaxFailureRule(prevRule);
+ @ThreadLeakZombies(Consequence.IGNORE_REMAINING_TESTS)
+ @ThreadLeakAction({ThreadLeakAction.Action.WARN})
+ @ThreadLeakScope(Scope.TEST)
+ @ThreadLeakLingering(linger = 500)
+ public static class Nested2 extends WithNestedTests.AbstractNestedTest {
+ public static final int TOTAL_ITERS = 10;
+ public static CountDownLatch die;
+ public static Thread zombie;
+ public static int testNum;
+
+ @BeforeClass
+ public static void setup() {
+ assert zombie == null;
+ die = new CountDownLatch(1);
+ testNum = 0;
+ }
+
+ @Repeat(iterations = TOTAL_ITERS)
+ public void testLeaveZombie() {
+ if (++testNum == 2) {
+ zombie = new Thread() {
+ @Override
+ public void run() {
+ while (true) {
+ try {
+ die.await();
+ return;
+ } catch (Exception e) { /* ignore */ }
+ }
+ }
+ };
+ zombie.start();
+ }
}
}
+
+ @Test
+ public void testZombieThreadFailures() throws Exception {
+ LuceneTestCase.replaceMaxFailureRule(new TestRuleIgnoreAfterMaxFailures(1));
+ JUnitCore core = new JUnitCore();
+ final StringBuilder results = new StringBuilder();
+ core.addListener(new RunListener() {
+ char lastTest;
+
+ @Override
+ public void testStarted(Description description) throws Exception {
+ lastTest = 'S'; // success.
+ }
+
+ @Override
+ public void testAssumptionFailure(Failure failure) {
+ lastTest = 'A'; // assumption failure.
+ }
+
+ @Override
+ public void testFailure(Failure failure) throws Exception {
+ lastTest = 'F'; // failure
+ System.out.println(failure.getMessage());
+ }
+
+ @Override
+ public void testFinished(Description description) throws Exception {
+ results.append(lastTest);
+ }
+ });
+
+ Result result = core.run(Nested2.class);
+ if (Nested2.die != null) {
+ Nested2.die.countDown();
+ Nested2.zombie.join();
+ }
+
+ super.prevSysOut.println(results.toString());
+ Assert.assertEquals(Nested2.TOTAL_ITERS, result.getRunCount());
+ Assert.assertEquals(results.toString(), "SFAAAAAAAA", results.toString());
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java b/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
index 425059a267b..221d361d570 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestOfflineSorter.java
@@ -21,6 +21,7 @@ import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
@@ -43,14 +44,14 @@ public class TestOfflineSorter extends LuceneTestCase {
public void setUp() throws Exception {
super.setUp();
tempDir = TestUtil.createTempDir("mergesort");
- TestUtil.rmDir(tempDir);
+ TestUtil.rm(tempDir);
tempDir.mkdirs();
}
@Override
public void tearDown() throws Exception {
if (tempDir != null)
- TestUtil.rmDir(tempDir);
+ TestUtil.rm(tempDir);
super.tearDown();
}
@@ -60,7 +61,7 @@ public class TestOfflineSorter extends LuceneTestCase {
public void testSingleLine() throws Exception {
checkSort(new OfflineSorter(), new byte [][] {
- "Single line only.".getBytes("UTF-8")
+ "Single line only.".getBytes(StandardCharsets.UTF_8)
});
}
diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java
index e435fe6f6ae..be40c6c00f1 100644
--- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java
+++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestDeterminizeLexicon.java
@@ -17,6 +17,7 @@ package org.apache.lucene.util.automaton;
* limitations under the License.
*/
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -56,7 +57,7 @@ public class TestDeterminizeLexicon extends LuceneTestCase {
}
final ByteRunAutomaton lexByte = new ByteRunAutomaton(lex);
for (String s : terms) {
- byte bytes[] = s.getBytes("UTF-8");
+ byte bytes[] = s.getBytes(StandardCharsets.UTF_8);
assertTrue(lexByte.run(bytes, 0, bytes.length));
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java b/lucene/core/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java
index 68c75a1f8cb..d5faa4dfef9 100644
--- a/lucene/core/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java
+++ b/lucene/core/src/test/org/apache/lucene/util/automaton/TestUTF32ToUTF8.java
@@ -22,6 +22,7 @@ import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.UnicodeUtil;
+import java.nio.charset.StandardCharsets;
import java.util.Random;
public class TestUTF32ToUTF8 extends LuceneTestCase {
@@ -184,7 +185,7 @@ public class TestUTF32ToUTF8 extends LuceneTestCase {
assertTrue(cra.run(input));
- byte[] bytes = input.getBytes("UTF-8");
+ byte[] bytes = input.getBytes(StandardCharsets.UTF_8);
assertTrue(bra.run(bytes, 0, bytes.length)); // this one fails!
}
@@ -197,7 +198,7 @@ public class TestUTF32ToUTF8 extends LuceneTestCase {
assertTrue(cra.run(input));
- byte[] bytes = input.getBytes("UTF-8");
+ byte[] bytes = input.getBytes(StandardCharsets.UTF_8);
assertTrue(bra.run(bytes, 0, bytes.length));
}
@@ -232,7 +233,7 @@ public class TestUTF32ToUTF8 extends LuceneTestCase {
throw e;
}
}
- byte bytes[] = string.getBytes("UTF-8");
+ byte bytes[] = string.getBytes(StandardCharsets.UTF_8);
assertEquals(cra.run(string), bra.run(bytes, 0, bytes.length));
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
index b8966e1427a..96d032fe758 100644
--- a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
+++ b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
@@ -63,6 +63,7 @@ import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.StringWriter;
import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -474,7 +475,7 @@ public class TestFSTs extends LuceneTestCase {
protected abstract T getOutput(IntsRef input, int ord) throws IOException;
public void run(int limit, boolean verify, boolean verifyByOutput) throws IOException {
- BufferedReader is = new BufferedReader(new InputStreamReader(new FileInputStream(wordsFileIn), "UTF-8"), 65536);
+ BufferedReader is = new BufferedReader(new InputStreamReader(new FileInputStream(wordsFileIn), StandardCharsets.UTF_8), 65536);
try {
final IntsRef intsRef = new IntsRef(10);
long tStart = System.currentTimeMillis();
@@ -517,7 +518,7 @@ public class TestFSTs extends LuceneTestCase {
System.out.println(ord + " terms; " + fst.getNodeCount() + " nodes; " + fst.getArcCount() + " arcs; " + fst.getArcWithOutputCount() + " arcs w/ output; tot size " + fst.sizeInBytes());
if (fst.getNodeCount() < 100) {
- Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"), "UTF-8");
+ Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"), StandardCharsets.UTF_8);
Util.toDot(fst, w, false, false);
w.close();
System.out.println("Wrote FST to out.dot");
@@ -544,7 +545,7 @@ public class TestFSTs extends LuceneTestCase {
while(true) {
for(int iter=0;iter<2;iter++) {
is.close();
- is = new BufferedReader(new InputStreamReader(new FileInputStream(wordsFileIn), "UTF-8"), 65536);
+ is = new BufferedReader(new InputStreamReader(new FileInputStream(wordsFileIn), StandardCharsets.UTF_8), 65536);
ord = 0;
tStart = System.currentTimeMillis();
diff --git a/lucene/core/src/test/org/apache/lucene/util/junitcompat/WithNestedTests.java b/lucene/core/src/test/org/apache/lucene/util/junitcompat/WithNestedTests.java
index 48792f175e6..a78c32b70c2 100644
--- a/lucene/core/src/test/org/apache/lucene/util/junitcompat/WithNestedTests.java
+++ b/lucene/core/src/test/org/apache/lucene/util/junitcompat/WithNestedTests.java
@@ -20,8 +20,11 @@ package org.apache.lucene.util.junitcompat;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
import java.util.List;
+import org.apache.lucene.util.FailureMarker;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures;
import org.apache.lucene.util.TestRuleIgnoreTestSuites;
@@ -35,6 +38,9 @@ import org.junit.Rule;
import org.junit.rules.RuleChain;
import org.junit.rules.TestRule;
+import com.carrotsearch.randomizedtesting.RandomizedRunner;
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import com.carrotsearch.randomizedtesting.SysGlobals;
import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;
import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
@@ -74,22 +80,36 @@ public abstract class WithNestedTests {
private TestRuleIgnoreAfterMaxFailures prevRule;
protected void before() throws Throwable {
- String filter = System.getProperty("tests.filter");
- if (filter != null && !filter.trim().isEmpty()) {
- // We're running with a complex test filter. This will affect nested tests anyway
- // so ignore them.
+ if (!isPropertyEmpty(SysGlobals.SYSPROP_TESTFILTER()) ||
+ !isPropertyEmpty(SysGlobals.SYSPROP_TESTCLASS()) ||
+ !isPropertyEmpty(SysGlobals.SYSPROP_TESTMETHOD()) ||
+ !isPropertyEmpty(SysGlobals.SYSPROP_ITERATIONS())) {
+ // We're running with a complex test filter that is properly handled by classes
+ // which are executed by RandomizedRunner. The "outer" classes testing LuceneTestCase
+ // itself are executed by the default JUnit runner and would be always executed.
+ // We thus always skip execution if any filtering is detected.
Assume.assumeTrue(false);
}
+ // Check zombie threads from previous suites. Don't run if zombies are around.
+ RandomizedTest.assumeFalse(RandomizedRunner.hasZombieThreads());
+
TestRuleIgnoreAfterMaxFailures newRule = new TestRuleIgnoreAfterMaxFailures(Integer.MAX_VALUE);
prevRule = LuceneTestCase.replaceMaxFailureRule(newRule);
+ RandomizedTest.assumeFalse(FailureMarker.hadFailures());
}
protected void afterAlways(List errors) throws Throwable {
if (prevRule != null) {
LuceneTestCase.replaceMaxFailureRule(prevRule);
}
+ FailureMarker.resetFailures();
}
+
+ private boolean isPropertyEmpty(String propertyName) {
+ String value = System.getProperty(propertyName);
+ return value == null || value.trim().isEmpty();
+ }
});
/**
@@ -121,14 +141,15 @@ public abstract class WithNestedTests {
try {
sysout = new ByteArrayOutputStream();
- System.setOut(new PrintStream(sysout, true, "UTF-8"));
+ System.setOut(new PrintStream(sysout, true, IOUtils.UTF_8));
syserr = new ByteArrayOutputStream();
- System.setErr(new PrintStream(syserr, true, "UTF-8"));
+ System.setErr(new PrintStream(syserr, true, IOUtils.UTF_8));
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
}
+ FailureMarker.resetFailures();
System.setProperty(TestRuleIgnoreTestSuites.PROPERTY_RUN_NESTED, "true");
}
@@ -146,20 +167,12 @@ public abstract class WithNestedTests {
protected String getSysOut() {
Assert.assertTrue(suppressOutputStreams);
System.out.flush();
- try {
- return new String(sysout.toByteArray(), "UTF-8");
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e);
- }
+ return new String(sysout.toByteArray(), StandardCharsets.UTF_8);
}
protected String getSysErr() {
Assert.assertTrue(suppressOutputStreams);
System.err.flush();
- try {
- return new String(syserr.toByteArray(), "UTF-8");
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e);
- }
+ return new String(syserr.toByteArray(), StandardCharsets.UTF_8);
}
}
diff --git a/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java b/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java
index 5f1e48d66e8..cfbfc43a8ab 100644
--- a/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java
+++ b/lucene/demo/src/java/org/apache/lucene/demo/IndexFiles.java
@@ -38,6 +38,7 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.util.Date;
/** Index all text files under a directory.
@@ -191,7 +192,7 @@ public class IndexFiles {
// so that the text of the file is tokenized and indexed, but not stored.
// Note that FileReader expects the file to be in UTF-8 encoding.
// If that's not the case searching for special characters will fail.
- doc.add(new TextField("contents", new BufferedReader(new InputStreamReader(fis, "UTF-8"))));
+ doc.add(new TextField("contents", new BufferedReader(new InputStreamReader(fis, StandardCharsets.UTF_8))));
if (writer.getConfig().getOpenMode() == OpenMode.CREATE) {
// New index, so we just add the document (no old document can be there):
diff --git a/lucene/demo/src/java/org/apache/lucene/demo/SearchFiles.java b/lucene/demo/src/java/org/apache/lucene/demo/SearchFiles.java
index 621ecf484c7..95b5975f4ec 100644
--- a/lucene/demo/src/java/org/apache/lucene/demo/SearchFiles.java
+++ b/lucene/demo/src/java/org/apache/lucene/demo/SearchFiles.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.util.Date;
import org.apache.lucene.analysis.Analyzer;
@@ -95,9 +96,9 @@ public class SearchFiles {
BufferedReader in = null;
if (queries != null) {
- in = new BufferedReader(new InputStreamReader(new FileInputStream(queries), "UTF-8"));
+ in = new BufferedReader(new InputStreamReader(new FileInputStream(queries), StandardCharsets.UTF_8));
} else {
- in = new BufferedReader(new InputStreamReader(System.in, "UTF-8"));
+ in = new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8));
}
// :Post-Release-Update-Version.LUCENE_XY:
QueryParser parser = new QueryParser(Version.LUCENE_50, field, analyzer);
diff --git a/lucene/demo/src/java/org/apache/lucene/demo/xmlparser/FormBasedXmlQueryDemo.java b/lucene/demo/src/java/org/apache/lucene/demo/xmlparser/FormBasedXmlQueryDemo.java
index bda9d3f1dad..79983eab4f5 100644
--- a/lucene/demo/src/java/org/apache/lucene/demo/xmlparser/FormBasedXmlQueryDemo.java
+++ b/lucene/demo/src/java/org/apache/lucene/demo/xmlparser/FormBasedXmlQueryDemo.java
@@ -21,9 +21,11 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.util.Enumeration;
import java.util.Properties;
import java.util.StringTokenizer;
+
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletConfig;
import javax.servlet.ServletException;
@@ -49,7 +51,6 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
/**
@@ -136,7 +137,7 @@ public class FormBasedXmlQueryDemo extends HttpServlet {
IndexWriterConfig iwConfig = new IndexWriterConfig(Version.LUCENE_CURRENT, analyzer);
IndexWriter writer = new IndexWriter(rd, iwConfig);
InputStream dataIn = getServletContext().getResourceAsStream("/WEB-INF/data.tsv");
- BufferedReader br = new BufferedReader(new InputStreamReader(dataIn, IOUtils.CHARSET_UTF_8));
+ BufferedReader br = new BufferedReader(new InputStreamReader(dataIn, StandardCharsets.UTF_8));
String line = br.readLine();
final FieldType textNoNorms = new FieldType(TextField.TYPE_STORED);
textNoNorms.setOmitNorms(true);
diff --git a/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java b/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
index 0e3f730bb13..8137057599a 100644
--- a/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
+++ b/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
@@ -22,6 +22,7 @@ import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
+import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.util.Arrays;
import java.util.Collections;
@@ -503,7 +504,7 @@ public class JavascriptCompiler {
try {
final Properties props = new Properties();
try (Reader in = IOUtils.getDecodingReader(JavascriptCompiler.class,
- JavascriptCompiler.class.getSimpleName() + ".properties", IOUtils.CHARSET_UTF_8)) {
+ JavascriptCompiler.class.getSimpleName() + ".properties", StandardCharsets.UTF_8)) {
props.load(in);
}
for (final String call : props.stringPropertyNames()) {
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/SlowRAMDirectory.java b/lucene/facet/src/test/org/apache/lucene/facet/SlowRAMDirectory.java
index ee21a8c211b..eaf78f7025d 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/SlowRAMDirectory.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/SlowRAMDirectory.java
@@ -167,6 +167,7 @@ public class SlowRAMDirectory extends RAMDirectory {
@Override public void flush() throws IOException { io.flush(); }
@Override public long getFilePointer() { return io.getFilePointer(); }
@Override public long length() throws IOException { return io.length(); }
+ @Override public long getChecksum() throws IOException { return io.getChecksum(); }
}
}
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java
index a63a27a6259..6f85c6ab12d 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java
@@ -132,8 +132,8 @@ public class TestTaxonomyFacetCounts extends FacetTestCase {
// Smoke test PrintTaxonomyStats:
ByteArrayOutputStream bos = new ByteArrayOutputStream();
- PrintTaxonomyStats.printStats(taxoReader, new PrintStream(bos, false, "UTF-8"), true);
- String result = bos.toString("UTF-8");
+ PrintTaxonomyStats.printStats(taxoReader, new PrintStream(bos, false, IOUtils.UTF_8), true);
+ String result = bos.toString(IOUtils.UTF_8);
assertTrue(result.indexOf("/Author: 4 immediate children; 5 total categories") != -1);
assertTrue(result.indexOf("/Publish Date: 3 immediate children; 12 total categories") != -1);
// Make sure at least a few nodes of the tree came out:
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestCharBlockArray.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestCharBlockArray.java
index 63aaeb42c1b..0a86e58ad84 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestCharBlockArray.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestCharBlockArray.java
@@ -8,11 +8,11 @@ import java.io.FileOutputStream;
import java.nio.ByteBuffer;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
import org.apache.lucene.facet.FacetTestCase;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.TestUtil;
+
import org.junit.Test;
/*
@@ -47,7 +47,7 @@ public class TestCharBlockArray extends FacetTestCase {
int size = 1 + random().nextInt(50);
// This test is turning random bytes into a string,
// this is asking for trouble.
- CharsetDecoder decoder = IOUtils.CHARSET_UTF_8.newDecoder()
+ CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder()
.onUnmappableCharacter(CodingErrorAction.REPLACE)
.onMalformedInput(CodingErrorAction.REPLACE);
String s = decoder.decode(ByteBuffer.wrap(buffer, 0, size)).toString();
@@ -60,7 +60,7 @@ public class TestCharBlockArray extends FacetTestCase {
int size = 1 + random().nextInt(50);
// This test is turning random bytes into a string,
// this is asking for trouble.
- CharsetDecoder decoder = IOUtils.CHARSET_UTF_8.newDecoder()
+ CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder()
.onUnmappableCharacter(CodingErrorAction.REPLACE)
.onMalformedInput(CodingErrorAction.REPLACE);
String s = decoder.decode(ByteBuffer.wrap(buffer, 0, size)).toString();
@@ -73,7 +73,7 @@ public class TestCharBlockArray extends FacetTestCase {
int size = 1 + random().nextInt(50);
// This test is turning random bytes into a string,
// this is asking for trouble.
- CharsetDecoder decoder = IOUtils.CHARSET_UTF_8.newDecoder()
+ CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder()
.onUnmappableCharacter(CodingErrorAction.REPLACE)
.onMalformedInput(CodingErrorAction.REPLACE);
String s = decoder.decode(ByteBuffer.wrap(buffer, 0, size)).toString();
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestCompactLabelToOrdinal.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestCompactLabelToOrdinal.java
index b5805e7e4ee..c29c849e56d 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestCompactLabelToOrdinal.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/writercache/TestCompactLabelToOrdinal.java
@@ -4,15 +4,15 @@ import java.io.File;
import java.nio.ByteBuffer;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.taxonomy.FacetLabel;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.TestUtil;
+
import org.junit.Test;
/*
@@ -53,7 +53,7 @@ public class TestCompactLabelToOrdinal extends FacetTestCase {
// This test is turning random bytes into a string,
// this is asking for trouble.
- CharsetDecoder decoder = IOUtils.CHARSET_UTF_8.newDecoder()
+ CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder()
.onUnmappableCharacter(CodingErrorAction.REPLACE)
.onMalformedInput(CodingErrorAction.REPLACE);
uniqueValues[i] = decoder.decode(ByteBuffer.wrap(buffer, 0, size)).toString();
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
index 706fcc63161..40c1c28183c 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
@@ -20,12 +20,14 @@ package org.apache.lucene.search.highlight;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.Reader;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
+
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -1512,7 +1514,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
// now an ugly built of XML parsing to test the snippet is encoded OK
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbf.newDocumentBuilder();
- org.w3c.dom.Document doc = db.parse(new ByteArrayInputStream(xhtml.getBytes("UTF-8")));
+ org.w3c.dom.Document doc = db.parse(new ByteArrayInputStream(xhtml.getBytes(StandardCharsets.UTF_8)));
Element root = doc.getDocumentElement();
NodeList nodes = root.getElementsByTagName("body");
Element body = (Element) nodes.item(0);
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java b/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java
index 29210a0223c..3ba2fef5f3b 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java
@@ -51,13 +51,11 @@ import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
/**
* Some tests that override {@link PostingsHighlighter#getIndexAnalyzer} to
* highlight wilcard, fuzzy, etc queries.
*/
-@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom"})
public class TestMultiTermHighlighting extends LuceneTestCase {
public void testWildcards() throws Exception {
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
index 9bb6f5349cb..69f4d9deee0 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java
@@ -20,6 +20,7 @@ package org.apache.lucene.search.postingshighlight;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.text.BreakIterator;
import java.util.Arrays;
import java.util.Map;
@@ -48,10 +49,8 @@ import org.apache.lucene.search.Sort;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.LuceneTestCase;
-@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom"})
public class TestPostingsHighlighter extends LuceneTestCase {
public void testBasics() throws Exception {
@@ -489,7 +488,7 @@ public class TestPostingsHighlighter extends LuceneTestCase {
public void testCambridgeMA() throws Exception {
BufferedReader r = new BufferedReader(new InputStreamReader(
- this.getClass().getResourceAsStream("CambridgeMA.utf8"), "UTF-8"));
+ this.getClass().getResourceAsStream("CambridgeMA.utf8"), StandardCharsets.UTF_8));
String text = r.readLine();
r.close();
Directory dir = newDirectory();
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java b/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java
index 3af9eb1cc38..7939b03409e 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighterRanking.java
@@ -43,10 +43,8 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.TestUtil;
-@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom"})
public class TestPostingsHighlighterRanking extends LuceneTestCase {
/**
* indexes a bunch of gibberish, and then highlights top(n).
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index 83c003d8237..93bbee7e01d 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -8,7 +8,7 @@
/cglib/cglib-nodep = 2.2
/com.adobe.xmp/xmpcore = 5.1.2
-com.carrotsearch.randomizedtesting.version = 2.1.1
+com.carrotsearch.randomizedtesting.version = 2.1.3
/com.carrotsearch.randomizedtesting/junit4-ant = ${com.carrotsearch.randomizedtesting.version}
/com.carrotsearch.randomizedtesting/randomizedtesting-runner = ${com.carrotsearch.randomizedtesting.version}
diff --git a/lucene/licenses/junit4-ant-2.1.1.jar.sha1 b/lucene/licenses/junit4-ant-2.1.1.jar.sha1
deleted file mode 100644
index 4340e4c8609..00000000000
--- a/lucene/licenses/junit4-ant-2.1.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a8a7371e11a8b3a4a3eeea81ad3cedafe3e3550e
diff --git a/lucene/licenses/junit4-ant-2.1.3.jar.sha1 b/lucene/licenses/junit4-ant-2.1.3.jar.sha1
new file mode 100644
index 00000000000..c2d6fa49fa8
--- /dev/null
+++ b/lucene/licenses/junit4-ant-2.1.3.jar.sha1
@@ -0,0 +1 @@
+8636804644d4ae3874f0efaa98978887e171cd55
diff --git a/lucene/licenses/randomizedtesting-runner-2.1.1.jar.sha1 b/lucene/licenses/randomizedtesting-runner-2.1.1.jar.sha1
deleted file mode 100644
index 2923eedf9fe..00000000000
--- a/lucene/licenses/randomizedtesting-runner-2.1.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5908c4e714dab40ccc892993a21537c7c0d6210c
diff --git a/lucene/licenses/randomizedtesting-runner-2.1.3.jar.sha1 b/lucene/licenses/randomizedtesting-runner-2.1.3.jar.sha1
new file mode 100644
index 00000000000..5da2ec2946a
--- /dev/null
+++ b/lucene/licenses/randomizedtesting-runner-2.1.3.jar.sha1
@@ -0,0 +1 @@
+d340caee99857ed0384681eea6219a4d937e7ee4
diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
index 87f33dc809f..db79ff021fd 100644
--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
@@ -794,6 +794,11 @@ public class MemoryIndex {
return null;
}
+ @Override
+ public void checkIntegrity() throws IOException {
+ // no-op
+ }
+
private class MemoryFields extends Fields {
@Override
public Iterator iterator() {
diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java b/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
index 093c0c36dc6..74318c81ecc 100644
--- a/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
@@ -21,6 +21,7 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.util.HashSet;
import java.util.Set;
@@ -100,7 +101,7 @@ public class MemoryIndexTest extends BaseTokenStreamTestCase {
private Set readQueries(String resource) throws IOException {
Set queries = new HashSet<>();
InputStream stream = getClass().getResourceAsStream(resource);
- BufferedReader reader = new BufferedReader(new InputStreamReader(stream, "UTF-8"));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8));
String line = null;
while ((line = reader.readLine()) != null) {
line = line.trim();
diff --git a/lucene/misc/build.xml b/lucene/misc/build.xml
index d49ccf77e19..b5ee7b2a5f4 100644
--- a/lucene/misc/build.xml
+++ b/lucene/misc/build.xml
@@ -44,7 +44,7 @@
+
diff --git a/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java b/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java
index 817b5620a89..4aee314ac8e 100644
--- a/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java
+++ b/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java
@@ -240,6 +240,11 @@ public class NativeUnixDirectory extends FSDirectory {
public long length() {
return fileLength + bufferPos;
}
+
+ @Override
+ public long getChecksum() throws IOException {
+ throw new UnsupportedOperationException("this directory currently does not work at all!");
+ }
@Override
public void close() throws IOException {
diff --git a/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java b/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java
index 49c036d5832..797a33d1e7f 100644
--- a/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java
+++ b/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java
@@ -172,12 +172,7 @@ public abstract class SorterTestBase extends LuceneTestCase {
doc.add(new StringField(ID_FIELD, Integer.toString(id), Store.YES));
doc.add(new StringField(DOCS_ENUM_FIELD, DOCS_ENUM_TERM, Store.NO));
positions.setId(id);
- if (doesntSupportOffsets.contains(TestUtil.getPostingsFormat(DOC_POSITIONS_FIELD))) {
- // codec doesnt support offsets: just index positions for the field
- doc.add(new Field(DOC_POSITIONS_FIELD, positions, TextField.TYPE_NOT_STORED));
- } else {
- doc.add(new Field(DOC_POSITIONS_FIELD, positions, POSITIONS_TYPE));
- }
+ doc.add(new Field(DOC_POSITIONS_FIELD, positions, POSITIONS_TYPE));
doc.add(new NumericDocValuesField(NUMERIC_DV_FIELD, id));
TextField norms = new TextField(NORMS_FIELD, Integer.toString(id), Store.NO);
norms.setBoost(Float.intBitsToFloat(id));
@@ -264,10 +259,8 @@ public abstract class SorterTestBase extends LuceneTestCase {
assertEquals("incorrect freq for doc=" + doc, sortedValues[doc].intValue() / 10 + 1, freq);
for (int i = 0; i < freq; i++) {
assertEquals("incorrect position for doc=" + doc, i, sortedPositions.nextPosition());
- if (!doesntSupportOffsets.contains(TestUtil.getPostingsFormat(DOC_POSITIONS_FIELD))) {
- assertEquals("incorrect startOffset for doc=" + doc, i, sortedPositions.startOffset());
- assertEquals("incorrect endOffset for doc=" + doc, i, sortedPositions.endOffset());
- }
+ assertEquals("incorrect startOffset for doc=" + doc, i, sortedPositions.startOffset());
+ assertEquals("incorrect endOffset for doc=" + doc, i, sortedPositions.endOffset());
assertEquals("incorrect payload for doc=" + doc, freq - i, Integer.parseInt(sortedPositions.getPayload().utf8ToString()));
}
}
@@ -284,10 +277,8 @@ public abstract class SorterTestBase extends LuceneTestCase {
assertEquals("incorrect freq for doc=" + doc, sortedValues[doc].intValue() / 10 + 1, freq);
for (int i = 0; i < freq; i++) {
assertEquals("incorrect position for doc=" + doc, i, sortedPositions.nextPosition());
- if (!doesntSupportOffsets.contains(TestUtil.getPostingsFormat(DOC_POSITIONS_FIELD))) {
- assertEquals("incorrect startOffset for doc=" + doc, i, sortedPositions.startOffset());
- assertEquals("incorrect endOffset for doc=" + doc, i, sortedPositions.endOffset());
- }
+ assertEquals("incorrect startOffset for doc=" + doc, i, sortedPositions.startOffset());
+ assertEquals("incorrect endOffset for doc=" + doc, i, sortedPositions.endOffset());
assertEquals("incorrect payload for doc=" + doc, freq - i, Integer.parseInt(sortedPositions.getPayload().utf8ToString()));
}
}
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java b/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java
index 728ac595cb3..68884892b97 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java
@@ -60,7 +60,6 @@ public class CommonTermsQueryTest extends LuceneTestCase {
public void testBasics() throws IOException {
Directory dir = newDirectory();
MockAnalyzer analyzer = new MockAnalyzer(random());
- analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));
RandomIndexWriter w = new RandomIndexWriter(random(), dir, analyzer);
String[] docs = new String[] {"this is the end of the world right",
"is this it or maybe not",
@@ -191,7 +190,6 @@ public class CommonTermsQueryTest extends LuceneTestCase {
public void testMinShouldMatch() throws IOException {
Directory dir = newDirectory();
MockAnalyzer analyzer = new MockAnalyzer(random());
- analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));
RandomIndexWriter w = new RandomIndexWriter(random(), dir, analyzer);
String[] docs = new String[] {"this is the end of the world right",
"is this it or maybe not",
@@ -351,7 +349,6 @@ public class CommonTermsQueryTest extends LuceneTestCase {
public void testExtend() throws IOException {
Directory dir = newDirectory();
MockAnalyzer analyzer = new MockAnalyzer(random());
- analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));
RandomIndexWriter w = new RandomIndexWriter(random(), dir, analyzer);
String[] docs = new String[] {"this is the end of the world right",
"is this it or maybe not",
diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java
index eaa71db5fe0..36e8b7e1b28 100644
--- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java
+++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java
@@ -45,6 +45,7 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.util.List;
@@ -63,7 +64,7 @@ public class TestParser extends LuceneTestCase {
builder = new CorePlusExtensionsParser("contents", analyzer);
BufferedReader d = new BufferedReader(new InputStreamReader(
- TestParser.class.getResourceAsStream("reuters21578.txt"), "US-ASCII"));
+ TestParser.class.getResourceAsStream("reuters21578.txt"), StandardCharsets.US_ASCII));
dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
String line = d.readLine();
diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java
index 727d5e10eac..3d05367f071 100644
--- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java
+++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java
@@ -33,9 +33,11 @@ import org.xml.sax.SAXException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
+
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
public class TestNumericRangeFilterBuilder extends LuceneTestCase {
@@ -203,7 +205,7 @@ public class TestNumericRangeFilterBuilder extends LuceneTestCase {
private static Document getDocumentFromString(String str)
throws SAXException, IOException, ParserConfigurationException {
- InputStream is = new ByteArrayInputStream(str.getBytes("UTF-8"));
+ InputStream is = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8));
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setNamespaceAware(true);
DocumentBuilder builder = factory.newDocumentBuilder();
diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeQueryBuilder.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeQueryBuilder.java
index 4ffee9d222a..53a7f09a4b2 100644
--- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeQueryBuilder.java
+++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeQueryBuilder.java
@@ -27,9 +27,11 @@ import org.xml.sax.SAXException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
+
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
public class TestNumericRangeQueryBuilder extends LuceneTestCase {
@@ -166,7 +168,7 @@ public class TestNumericRangeQueryBuilder extends LuceneTestCase {
private static Document getDocumentFromString(String str)
throws SAXException, IOException, ParserConfigurationException {
- InputStream is = new ByteArrayInputStream(str.getBytes("UTF-8"));
+ InputStream is = new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8));
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setNamespaceAware(true);
DocumentBuilder builder = factory.newDocumentBuilder();
diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery2.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery2.java
index c93c14a057c..178d3dac89e 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery2.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery2.java
@@ -20,6 +20,7 @@ package org.apache.lucene.sandbox.queries;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
@@ -83,7 +84,7 @@ public class TestSlowFuzzyQuery2 extends LuceneTestCase {
System.out.println("TEST: codePointTable=" + codePointTable);
}
InputStream stream = getClass().getResourceAsStream("fuzzyTestData.txt");
- BufferedReader reader = new BufferedReader(new InputStreamReader(stream, "UTF-8"));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8));
int bits = Integer.parseInt(reader.readLine());
int terms = (int) Math.pow(2, bits);
diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java b/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java
index 7ac787e5e1b..64c814786bc 100644
--- a/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java
+++ b/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java
@@ -23,6 +23,7 @@ import com.spatial4j.core.shape.Rectangle;
import com.spatial4j.core.shape.Shape;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -41,7 +42,7 @@ import java.util.List;
*/
public abstract class SpatialPrefixTree {
- protected static final Charset UTF8 = Charset.forName("UTF-8");
+ protected static final Charset UTF8 = StandardCharsets.UTF_8;
protected final int maxLevels;
diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestData.java b/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestData.java
index eeaea3081c9..fdf13cd7ec5 100644
--- a/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestData.java
+++ b/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestData.java
@@ -24,6 +24,7 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Iterator;
@@ -43,7 +44,7 @@ public class SpatialTestData {
*/
public static Iterator getTestData(InputStream in, SpatialContext ctx) throws IOException {
List results = new ArrayList<>();
- BufferedReader bufInput = new BufferedReader(new InputStreamReader(in,"UTF-8"));
+ BufferedReader bufInput = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
try {
String line;
while ((line = bufInput.readLine()) != null) {
diff --git a/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestQuery.java b/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestQuery.java
index 4ccf469c022..a73baa9585a 100644
--- a/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestQuery.java
+++ b/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestQuery.java
@@ -18,6 +18,7 @@ package org.apache.lucene.spatial;
*/
import com.spatial4j.core.context.SpatialContext;
+
import org.apache.lucene.spatial.query.SpatialArgs;
import org.apache.lucene.spatial.query.SpatialArgsParser;
@@ -25,6 +26,7 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -51,7 +53,7 @@ public class SpatialTestQuery {
List results = new ArrayList<>();
- BufferedReader bufInput = new BufferedReader(new InputStreamReader(in,"UTF-8"));
+ BufferedReader bufInput = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
try {
String line;
for (int lineNumber = 1; (line = bufInput.readLine()) != null; lineNumber++) {
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java
index 5e77021af79..91f9db6328c 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
+import java.nio.charset.StandardCharsets;
import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.util.BytesRef;
@@ -47,7 +48,7 @@ public class PlainTextDictionary implements Dictionary {
* NOTE: content is treated as UTF-8
*/
public PlainTextDictionary(File file) throws IOException {
- in = new BufferedReader(IOUtils.getDecodingReader(file, IOUtils.CHARSET_UTF_8));
+ in = new BufferedReader(IOUtils.getDecodingReader(file, StandardCharsets.UTF_8));
}
/**
@@ -56,7 +57,7 @@ public class PlainTextDictionary implements Dictionary {
* NOTE: content is treated as UTF-8
*/
public PlainTextDictionary(InputStream dictFile) {
- in = new BufferedReader(IOUtils.getDecodingReader(dictFile, IOUtils.CHARSET_UTF_8));
+ in = new BufferedReader(IOUtils.getDecodingReader(dictFile, StandardCharsets.UTF_8));
}
/**
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java
index 1d24c2c9d8c..5006cb4e8d9 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java
@@ -19,6 +19,7 @@ package org.apache.lucene.search.suggest;
import java.io.*;
+import java.nio.charset.StandardCharsets;
import java.util.Set;
import org.apache.lucene.search.spell.Dictionary;
@@ -102,7 +103,7 @@ public class FileDictionary implements Dictionary {
* NOTE: content is treated as UTF-8
*/
public FileDictionary(InputStream dictFile, String fieldDelimiter) {
- in = new BufferedReader(IOUtils.getDecodingReader(dictFile, IOUtils.CHARSET_UTF_8));
+ in = new BufferedReader(IOUtils.getDecodingReader(dictFile, StandardCharsets.UTF_8));
this.fieldDelimiter = fieldDelimiter;
}
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java
index e0c875a3c1a..d340134d59b 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java
@@ -209,7 +209,7 @@ public class SortedInputIterator implements InputIterator {
}
}
- /** encodes an entry (bytes+(payload)+(contexts)+weight) to the provided writer */
+ /** encodes an entry (bytes+(contexts)+(payload)+weight) to the provided writer */
protected void encode(ByteSequencesWriter writer, ByteArrayDataOutput output, byte[] buffer, BytesRef spare, BytesRef payload, Set contexts, long weight) throws IOException {
int requiredLength = spare.length + 8 + ((hasPayloads) ? 2 + payload.length : 0);
if (hasContexts) {
@@ -223,10 +223,6 @@ public class SortedInputIterator implements InputIterator {
}
output.reset(buffer);
output.writeBytes(spare.bytes, spare.offset, spare.length);
- if (hasPayloads) {
- output.writeBytes(payload.bytes, payload.offset, payload.length);
- output.writeShort((short) payload.length);
- }
if (hasContexts) {
for (BytesRef ctx : contexts) {
output.writeBytes(ctx.bytes, ctx.offset, ctx.length);
@@ -234,6 +230,10 @@ public class SortedInputIterator implements InputIterator {
}
output.writeShort((short) contexts.size());
}
+ if (hasPayloads) {
+ output.writeBytes(payload.bytes, payload.offset, payload.length);
+ output.writeShort((short) payload.length);
+ }
output.writeLong(weight);
writer.write(buffer, 0, output.getPosition());
}
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FuzzySuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FuzzySuggester.java
index 63a26e20245..458d59d5ba9 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FuzzySuggester.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FuzzySuggester.java
@@ -17,6 +17,7 @@ package org.apache.lucene.search.suggest.analyzing;
*/
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
@@ -192,7 +193,7 @@ public final class FuzzySuggester extends AnalyzingSuggester {
Automaton levA = convertAutomaton(toLevenshteinAutomata(lookupAutomaton));
/*
- Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"), "UTF-8");
+ Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"), StandardCharsets.UTF_8);
w.write(levA.toDot());
w.close();
System.out.println("Wrote LevA to out.dot");
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java
index 2bffc11db35..d3e106cd19f 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java
@@ -33,7 +33,7 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
-import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Locale;
import java.util.Vector;
@@ -224,15 +224,13 @@ public class JaspellTernarySearchTrie {
BufferedReader in;
if (compression)
in = new BufferedReader(IOUtils.getDecodingReader(new GZIPInputStream(
- new FileInputStream(file)), IOUtils.CHARSET_UTF_8));
+ new FileInputStream(file)), StandardCharsets.UTF_8));
else in = new BufferedReader(IOUtils.getDecodingReader((new FileInputStream(
- file)), IOUtils.CHARSET_UTF_8));
+ file)), StandardCharsets.UTF_8));
String word;
int pos;
Float occur, one = new Float(1);
- int numWords = 0;
while ((word = in.readLine()) != null) {
- numWords++;
pos = word.indexOf("\t");
occur = one;
if (pos != -1) {
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/FileDictionaryTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/FileDictionaryTest.java
index 2d880f587a5..29f7387fca7 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/FileDictionaryTest.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/FileDictionaryTest.java
@@ -3,6 +3,7 @@ package org.apache.lucene.search.suggest;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
import java.util.AbstractMap.SimpleEntry;
import java.util.ArrayList;
import java.util.List;
@@ -74,7 +75,7 @@ public class FileDictionaryTest extends LuceneTestCase {
@Test
public void testFileWithTerm() throws IOException {
Map.Entry>,String> fileInput = generateFileInput(atLeast(100), FileDictionary.DEFAULT_FIELD_DELIMITER, false, false);
- InputStream inputReader = new ByteArrayInputStream(fileInput.getValue().getBytes("UTF-8"));
+ InputStream inputReader = new ByteArrayInputStream(fileInput.getValue().getBytes(StandardCharsets.UTF_8));
FileDictionary dictionary = new FileDictionary(inputReader);
List> entries = fileInput.getKey();
InputIterator inputIter = dictionary.getEntryIterator();
@@ -96,7 +97,7 @@ public class FileDictionaryTest extends LuceneTestCase {
@Test
public void testFileWithWeight() throws IOException {
Map.Entry>,String> fileInput = generateFileInput(atLeast(100), FileDictionary.DEFAULT_FIELD_DELIMITER, true, false);
- InputStream inputReader = new ByteArrayInputStream(fileInput.getValue().getBytes("UTF-8"));
+ InputStream inputReader = new ByteArrayInputStream(fileInput.getValue().getBytes(StandardCharsets.UTF_8));
FileDictionary dictionary = new FileDictionary(inputReader);
List> entries = fileInput.getKey();
InputIterator inputIter = dictionary.getEntryIterator();
@@ -118,7 +119,7 @@ public class FileDictionaryTest extends LuceneTestCase {
@Test
public void testFileWithWeightAndPayload() throws IOException {
Map.Entry>,String> fileInput = generateFileInput(atLeast(100), FileDictionary.DEFAULT_FIELD_DELIMITER, true, true);
- InputStream inputReader = new ByteArrayInputStream(fileInput.getValue().getBytes("UTF-8"));
+ InputStream inputReader = new ByteArrayInputStream(fileInput.getValue().getBytes(StandardCharsets.UTF_8));
FileDictionary dictionary = new FileDictionary(inputReader);
List> entries = fileInput.getKey();
InputIterator inputIter = dictionary.getEntryIterator();
@@ -144,7 +145,7 @@ public class FileDictionaryTest extends LuceneTestCase {
@Test
public void testFileWithOneEntry() throws IOException {
Map.Entry>,String> fileInput = generateFileInput(1, FileDictionary.DEFAULT_FIELD_DELIMITER, true, true);
- InputStream inputReader = new ByteArrayInputStream(fileInput.getValue().getBytes("UTF-8"));
+ InputStream inputReader = new ByteArrayInputStream(fileInput.getValue().getBytes(StandardCharsets.UTF_8));
FileDictionary dictionary = new FileDictionary(inputReader);
List> entries = fileInput.getKey();
InputIterator inputIter = dictionary.getEntryIterator();
@@ -171,7 +172,7 @@ public class FileDictionaryTest extends LuceneTestCase {
@Test
public void testFileWithDifferentDelimiter() throws IOException {
Map.Entry>,String> fileInput = generateFileInput(atLeast(100), " , ", true, true);
- InputStream inputReader = new ByteArrayInputStream(fileInput.getValue().getBytes("UTF-8"));
+ InputStream inputReader = new ByteArrayInputStream(fileInput.getValue().getBytes(StandardCharsets.UTF_8));
FileDictionary dictionary = new FileDictionary(inputReader, " , ");
List> entries = fileInput.getKey();
InputIterator inputIter = dictionary.getEntryIterator();
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java
index 3bcedab6d50..1efebbefa0b 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java
@@ -22,6 +22,7 @@ import java.io.InputStreamReader;
import java.lang.reflect.Constructor;
import java.net.URL;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -92,7 +93,7 @@ public class LookupBenchmarkTest extends LuceneTestCase {
LookupBenchmarkTest.benchmarkInput = input;
}
- static final Charset UTF_8 = Charset.forName("UTF-8");
+ static final Charset UTF_8 = StandardCharsets.UTF_8;
/**
* Collect the multilingual input for benchmarks/ tests.
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java
index 2b7cb67d186..3aff3c21d36 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java
@@ -48,9 +48,11 @@ public class TestInputIterator extends LuceneTestCase {
TreeMap> sorted = new TreeMap<>(comparator);
TreeMap sortedWithoutPayload = new TreeMap<>(comparator);
TreeMap>> sortedWithContext = new TreeMap<>(comparator);
+ TreeMap>>> sortedWithPayloadAndContext = new TreeMap<>(comparator);
Input[] unsorted = new Input[num];
Input[] unsortedWithoutPayload = new Input[num];
Input[] unsortedWithContexts = new Input[num];
+ Input[] unsortedWithPayloadAndContext = new Input[num];
Set ctxs;
for (int i = 0; i < num; i++) {
BytesRef key;
@@ -67,9 +69,11 @@ public class TestInputIterator extends LuceneTestCase {
sortedWithoutPayload.put(key, value);
sorted.put(key, new SimpleEntry<>(value, payload));
sortedWithContext.put(key, new SimpleEntry<>(value, ctxs));
+ sortedWithPayloadAndContext.put(key, new SimpleEntry<>(value, new SimpleEntry<>(payload, ctxs)));
unsorted[i] = new Input(key, value, payload);
unsortedWithoutPayload[i] = new Input(key, value);
unsortedWithContexts[i] = new Input(key, value, ctxs);
+ unsortedWithPayloadAndContext[i] = new Input(key, value, payload, ctxs);
}
// test the sorted iterator wrapper with payloads
@@ -96,6 +100,20 @@ public class TestInputIterator extends LuceneTestCase {
}
assertNull(wrapper.next());
+ // test the sorted iterator wrapper with contexts and payload
+ wrapper = new SortedInputIterator(new InputArrayIterator(unsortedWithPayloadAndContext), comparator);
+ Iterator>>>> expectedPayloadContextEntries = sortedWithPayloadAndContext.entrySet().iterator();
+ while (expectedPayloadContextEntries.hasNext()) {
+ Map.Entry>>> entry = expectedPayloadContextEntries.next();
+ assertEquals(entry.getKey(), wrapper.next());
+ assertEquals(entry.getValue().getKey().longValue(), wrapper.weight());
+ Set actualCtxs = entry.getValue().getValue().getValue();
+ assertEquals(actualCtxs, wrapper.contexts());
+ BytesRef actualPayload = entry.getValue().getValue().getKey();
+ assertEquals(actualPayload, wrapper.payload());
+ }
+ assertNull(wrapper.next());
+
// test the unsorted iterator wrapper with payloads
wrapper = new UnsortedInputIterator(new InputArrayIterator(unsorted));
TreeMap> actual = new TreeMap<>();
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java
index 1de609a4d2d..19a4d2b825f 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java
@@ -39,12 +39,9 @@ import org.apache.lucene.search.suggest.Input;
import org.apache.lucene.search.suggest.InputArrayIterator;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
-// Test requires postings offsets:
-@SuppressCodecs({"Lucene3x","MockFixedIntBlock","MockVariableIntBlock","MockSep","MockRandom"})
public class AnalyzingInfixSuggesterTest extends LuceneTestCase {
public void testBasic() throws Exception {
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
index 1b027948ce8..acade2e764e 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
@@ -17,6 +17,7 @@ package org.apache.lucene.search.suggest.fst;
* limitations under the License.
*/
+import java.nio.charset.StandardCharsets;
import java.util.*;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
@@ -144,7 +145,7 @@ public class FSTCompletionTest extends LuceneTestCase {
public void testThreeByte() throws Exception {
String key = new String(new byte[] {
- (byte) 0xF0, (byte) 0xA4, (byte) 0xAD, (byte) 0xA2}, "UTF-8");
+ (byte) 0xF0, (byte) 0xA4, (byte) 0xAD, (byte) 0xA2}, StandardCharsets.UTF_8);
FSTCompletionBuilder builder = new FSTCompletionBuilder();
builder.add(new BytesRef(key), 0);
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java
index 0cb6c668d02..d55e8febb64 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java
@@ -23,6 +23,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.OfflineSorter;
@@ -43,7 +44,7 @@ public class LargeInputFST {
BufferedReader reader = new BufferedReader(
new InputStreamReader(
- new FileInputStream(input), "UTF-8"));
+ new FileInputStream(input), StandardCharsets.UTF_8));
BytesRef scratch = new BytesRef();
String line;
diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
index da0ec735601..48cd8298541 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
@@ -25,6 +25,7 @@ import java.io.Reader;
import java.io.StringReader;
import java.io.StringWriter;
import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.CountDownLatch;
@@ -562,14 +563,12 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase {
if (random.nextBoolean()) {
ft.setOmitNorms(true);
}
- String pf = TestUtil.getPostingsFormat("dummy");
- boolean supportsOffsets = !doesntSupportOffsets.contains(pf);
switch(random.nextInt(4)) {
case 0: ft.setIndexOptions(IndexOptions.DOCS_ONLY); break;
case 1: ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS); break;
case 2: ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); break;
default:
- if (supportsOffsets && offsetsAreCorrect) {
+ if (offsetsAreCorrect) {
ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
} else {
ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
@@ -892,7 +891,7 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase {
}
protected void toDotFile(Analyzer a, String inputText, String localFileName) throws IOException {
- Writer w = new OutputStreamWriter(new FileOutputStream(localFileName), "UTF-8");
+ Writer w = new OutputStreamWriter(new FileOutputStream(localFileName), StandardCharsets.UTF_8);
final TokenStream ts = a.tokenStream("field", inputText);
ts.reset();
new TokenStreamToDot(inputText, ts, new PrintWriter(w)).toDot();
diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java
index b144f935040..09393ef6d69 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java
@@ -24,6 +24,7 @@ import org.apache.lucene.util.BytesRef;
import java.io.IOException;
import java.io.Reader;
+import java.nio.charset.StandardCharsets;
/**
@@ -68,7 +69,7 @@ final class MockPayloadFilter extends TokenFilter {
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
- payloadAttr.setPayload(new BytesRef(("pos: " + pos).getBytes("UTF-8")));
+ payloadAttr.setPayload(new BytesRef(("pos: " + pos).getBytes(StandardCharsets.UTF_8)));
int posIncr;
if (pos == 0 || i % 2 == 1) {
posIncr = 1;
diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/VocabularyAssert.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/VocabularyAssert.java
index 659831856c3..cdcacc587bf 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/analysis/VocabularyAssert.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/VocabularyAssert.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.util.zip.ZipFile;
import org.apache.lucene.analysis.Analyzer;
@@ -34,9 +35,9 @@ public class VocabularyAssert {
public static void assertVocabulary(Analyzer a, InputStream voc, InputStream out)
throws IOException {
BufferedReader vocReader = new BufferedReader(
- new InputStreamReader(voc, "UTF-8"));
+ new InputStreamReader(voc, StandardCharsets.UTF_8));
BufferedReader outputReader = new BufferedReader(
- new InputStreamReader(out, "UTF-8"));
+ new InputStreamReader(out, StandardCharsets.UTF_8));
String inputWord = null;
while ((inputWord = vocReader.readLine()) != null) {
String expectedWord = outputReader.readLine();
@@ -49,7 +50,7 @@ public class VocabularyAssert {
public static void assertVocabulary(Analyzer a, InputStream vocOut)
throws IOException {
BufferedReader vocReader = new BufferedReader(
- new InputStreamReader(vocOut, "UTF-8"));
+ new InputStreamReader(vocOut, StandardCharsets.UTF_8));
String inputLine = null;
while ((inputLine = vocReader.readLine()) != null) {
if (inputLine.startsWith("#") || inputLine.trim().length() == 0)
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
index 512f92596d2..42484f5c469 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
@@ -306,5 +306,10 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
public long ramBytesUsed() {
return in.ramBytesUsed();
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ in.checkIntegrity();
+ }
}
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
index 8681b59a505..d9bdc1c48dc 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
@@ -89,6 +89,11 @@ public final class AssertingPostingsFormat extends PostingsFormat {
public long ramBytesUsed() {
return in.ramBytesUsed();
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ in.checkIntegrity();
+ }
}
static class AssertingFieldsConsumer extends FieldsConsumer {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java
index a2763875662..c58689bda79 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java
@@ -76,6 +76,11 @@ public class AssertingStoredFieldsFormat extends StoredFieldsFormat {
public long ramBytesUsed() {
return in.ramBytesUsed();
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ in.checkIntegrity();
+ }
}
enum Status {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
index d6503f4d039..7119e595a04 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
@@ -75,6 +75,11 @@ public class AssertingTermVectorsFormat extends TermVectorsFormat {
public long ramBytesUsed() {
return in.ramBytesUsed();
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ in.checkIntegrity();
+ }
}
enum Status {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java
index 78680f1dd33..999014a56c7 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java
@@ -46,7 +46,7 @@ import org.apache.lucene.util.packed.MonotonicBlockPackedWriter;
import org.apache.lucene.util.packed.PackedInts.FormatAndBits;
import org.apache.lucene.util.packed.PackedInts;
-import static org.apache.lucene.codecs.lucene42.Lucene42DocValuesProducer.VERSION_CURRENT;
+import static org.apache.lucene.codecs.lucene42.Lucene42DocValuesProducer.VERSION_GCD_COMPRESSION;
import static org.apache.lucene.codecs.lucene42.Lucene42DocValuesProducer.BLOCK_SIZE;
import static org.apache.lucene.codecs.lucene42.Lucene42DocValuesProducer.BYTES;
import static org.apache.lucene.codecs.lucene42.Lucene42DocValuesProducer.NUMBER;
@@ -71,10 +71,11 @@ class Lucene42DocValuesConsumer extends DocValuesConsumer {
try {
String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
data = state.directory.createOutput(dataName, state.context);
- CodecUtil.writeHeader(data, dataCodec, VERSION_CURRENT);
+ // this writer writes the format 4.2 did!
+ CodecUtil.writeHeader(data, dataCodec, VERSION_GCD_COMPRESSION);
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
meta = state.directory.createOutput(metaName, state.context);
- CodecUtil.writeHeader(meta, metaCodec, VERSION_CURRENT);
+ CodecUtil.writeHeader(meta, metaCodec, VERSION_GCD_COMPRESSION);
success = true;
} finally {
if (!success) {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java
deleted file mode 100644
index b714489fb9f..00000000000
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java
+++ /dev/null
@@ -1,202 +0,0 @@
-package org.apache.lucene.codecs.mockintblock;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-
-import org.apache.lucene.codecs.FieldsConsumer;
-import org.apache.lucene.codecs.FieldsProducer;
-import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.blockterms.BlockTermsReader;
-import org.apache.lucene.codecs.blockterms.BlockTermsWriter;
-import org.apache.lucene.codecs.blockterms.FixedGapTermsIndexReader;
-import org.apache.lucene.codecs.blockterms.FixedGapTermsIndexWriter;
-import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;
-import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;
-import org.apache.lucene.codecs.intblock.FixedIntBlockIndexInput;
-import org.apache.lucene.codecs.intblock.FixedIntBlockIndexOutput;
-import org.apache.lucene.codecs.sep.IntIndexInput;
-import org.apache.lucene.codecs.sep.IntIndexOutput;
-import org.apache.lucene.codecs.sep.IntStreamFactory;
-import org.apache.lucene.codecs.sep.SepPostingsReader;
-import org.apache.lucene.codecs.sep.SepPostingsWriter;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.store.*;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-
-/**
- * A silly test codec to verify core support for fixed
- * sized int block encoders is working. The int encoder
- * used here just writes each block as a series of vInt.
- */
-
-public final class MockFixedIntBlockPostingsFormat extends PostingsFormat {
-
- private final int blockSize;
-
- public MockFixedIntBlockPostingsFormat() {
- this(1);
- }
-
- public MockFixedIntBlockPostingsFormat(int blockSize) {
- super("MockFixedIntBlock");
- this.blockSize = blockSize;
- }
-
- @Override
- public String toString() {
- return getName() + "(blockSize=" + blockSize + ")";
- }
-
- // only for testing
- public IntStreamFactory getIntFactory() {
- return new MockIntFactory(blockSize);
- }
-
- /**
- * Encodes blocks as vInts of a fixed block size.
- */
- public static class MockIntFactory extends IntStreamFactory {
- private final int blockSize;
-
- public MockIntFactory(int blockSize) {
- this.blockSize = blockSize;
- }
-
- @Override
- public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException {
- return new FixedIntBlockIndexInput(dir.openInput(fileName, context)) {
-
- @Override
- protected BlockReader getBlockReader(final IndexInput in, final int[] buffer) {
- return new BlockReader() {
- public void seek(long pos) {}
- @Override
- public void readBlock() throws IOException {
- for(int i=0;i= count: "buffer.length=" + buffer.length + " count=" + count;
- for(int i=0;i
-
-
-
- delegates = new ArrayList<>();
-
- public MockIntStreamFactory(Random random) {
- salt = random.nextInt();
- delegates.add(new MockSingleIntFactory());
- final int blockSize = TestUtil.nextInt(random, 1, 2000);
- delegates.add(new MockFixedIntBlockPostingsFormat.MockIntFactory(blockSize));
- final int baseBlockSize = TestUtil.nextInt(random, 1, 127);
- delegates.add(new MockVariableIntBlockPostingsFormat.MockIntFactory(baseBlockSize));
- // TODO: others
- }
-
- private static String getExtension(String fileName) {
- final int idx = fileName.indexOf('.');
- assert idx != -1;
- return fileName.substring(idx);
- }
-
- @Override
- public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException {
- // Must only use extension, because IW.addIndexes can
- // rename segment!
- final IntStreamFactory f = delegates.get((Math.abs(salt ^ getExtension(fileName).hashCode())) % delegates.size());
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: read using int factory " + f + " from fileName=" + fileName);
- }
- return f.openInput(dir, fileName, context);
- }
-
- @Override
- public IntIndexOutput createOutput(Directory dir, String fileName, IOContext context) throws IOException {
- final IntStreamFactory f = delegates.get((Math.abs(salt ^ getExtension(fileName).hashCode())) % delegates.size());
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: write using int factory " + f + " to fileName=" + fileName);
- }
- return f.createOutput(dir, fileName, context);
- }
- }
-
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
int minSkipInterval;
@@ -171,16 +117,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
random.nextInt(); // consume a random for buffersize
- PostingsWriterBase postingsWriter;
- if (random.nextBoolean()) {
- postingsWriter = new SepPostingsWriter(state, new MockIntStreamFactory(random), skipInterval);
- } else {
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: writing Standard postings");
- }
- // TODO: randomize variables like acceptibleOverHead?!
- postingsWriter = new Lucene41PostingsWriter(state, skipInterval);
- }
+ PostingsWriterBase postingsWriter = new Lucene41PostingsWriter(state, skipInterval);
if (random.nextBoolean()) {
final int totTFCutoff = TestUtil.nextInt(random, 1, 20);
@@ -327,20 +264,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
System.out.println("MockRandomCodec: readBufferSize=" + readBufferSize);
}
- PostingsReaderBase postingsReader;
-
- if (random.nextBoolean()) {
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: reading Sep postings");
- }
- postingsReader = new SepPostingsReader(state.directory, state.fieldInfos, state.segmentInfo,
- state.context, new MockIntStreamFactory(random), state.segmentSuffix);
- } else {
- if (LuceneTestCase.VERBOSE) {
- System.out.println("MockRandomCodec: reading Standard postings");
- }
- postingsReader = new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
- }
+ PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
if (random.nextBoolean()) {
final int totTFCutoff = TestUtil.nextInt(random, 1, 20);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSepPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSepPostingsFormat.java
deleted file mode 100644
index 9fc3cd891d9..00000000000
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSepPostingsFormat.java
+++ /dev/null
@@ -1,125 +0,0 @@
-package org.apache.lucene.codecs.mocksep;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-
-import org.apache.lucene.codecs.FieldsConsumer;
-import org.apache.lucene.codecs.FieldsProducer;
-import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.blockterms.BlockTermsReader;
-import org.apache.lucene.codecs.blockterms.BlockTermsWriter;
-import org.apache.lucene.codecs.blockterms.FixedGapTermsIndexReader;
-import org.apache.lucene.codecs.blockterms.FixedGapTermsIndexWriter;
-import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;
-import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;
-import org.apache.lucene.codecs.sep.SepPostingsReader;
-import org.apache.lucene.codecs.sep.SepPostingsWriter;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.BytesRef;
-
-/**
- * A silly codec that simply writes each file separately as
- * single vInts. Don't use this (performance will be poor)!
- * This is here just to test the core sep codec
- * classes.
- */
-public final class MockSepPostingsFormat extends PostingsFormat {
-
- public MockSepPostingsFormat() {
- super("MockSep");
- }
-
- @Override
- public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
-
- PostingsWriterBase postingsWriter = new SepPostingsWriter(state, new MockSingleIntFactory());
-
- boolean success = false;
- TermsIndexWriterBase indexWriter;
- try {
- indexWriter = new FixedGapTermsIndexWriter(state);
- success = true;
- } finally {
- if (!success) {
- postingsWriter.close();
- }
- }
-
- success = false;
- try {
- FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, postingsWriter);
- success = true;
- return ret;
- } finally {
- if (!success) {
- try {
- postingsWriter.close();
- } finally {
- indexWriter.close();
- }
- }
- }
- }
-
- @Override
- public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
-
- PostingsReaderBase postingsReader = new SepPostingsReader(state.directory, state.fieldInfos, state.segmentInfo,
- state.context, new MockSingleIntFactory(), state.segmentSuffix);
-
- TermsIndexReaderBase indexReader;
- boolean success = false;
- try {
- indexReader = new FixedGapTermsIndexReader(state.directory,
- state.fieldInfos,
- state.segmentInfo.name,
- BytesRef.getUTF8SortedAsUnicodeComparator(),
- state.segmentSuffix, state.context);
- success = true;
- } finally {
- if (!success) {
- postingsReader.close();
- }
- }
-
- success = false;
- try {
- FieldsProducer ret = new BlockTermsReader(indexReader,
- state.directory,
- state.fieldInfos,
- state.segmentInfo,
- postingsReader,
- state.context,
- state.segmentSuffix);
- success = true;
- return ret;
- } finally {
- if (!success) {
- try {
- postingsReader.close();
- } finally {
- indexReader.close();
- }
- }
- }
- }
-}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntFactory.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntFactory.java
deleted file mode 100644
index ca42debdca2..00000000000
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntFactory.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package org.apache.lucene.codecs.mocksep;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.codecs.sep.IntIndexInput;
-import org.apache.lucene.codecs.sep.IntIndexOutput;
-import org.apache.lucene.codecs.sep.IntStreamFactory;
-
-import java.io.IOException;
-
-/**
- * Encodes ints directly as vInts with {@link MockSingleIntIndexOutput}
- * @lucene.experimental
- */
-public class MockSingleIntFactory extends IntStreamFactory {
- @Override
- public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException {
- return new MockSingleIntIndexInput(dir, fileName, context);
- }
- @Override
- public IntIndexOutput createOutput(Directory dir, String fileName, IOContext context) throws IOException {
- return new MockSingleIntIndexOutput(dir, fileName, context);
- }
-}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntIndexInput.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntIndexInput.java
deleted file mode 100644
index 765aeabbbee..00000000000
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntIndexInput.java
+++ /dev/null
@@ -1,117 +0,0 @@
-package org.apache.lucene.codecs.mocksep;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.sep.IntIndexInput;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-
-/** Reads IndexInputs written with {@link
- * MockSingleIntIndexOutput}. NOTE: this class is just for
- * demonstration purposes (it is a very slow way to read a
- * block of ints).
- *
- * @lucene.experimental
- */
-public class MockSingleIntIndexInput extends IntIndexInput {
- private final IndexInput in;
-
- public MockSingleIntIndexInput(Directory dir, String fileName, IOContext context)
- throws IOException {
- in = dir.openInput(fileName, context);
- CodecUtil.checkHeader(in, MockSingleIntIndexOutput.CODEC,
- MockSingleIntIndexOutput.VERSION_START,
- MockSingleIntIndexOutput.VERSION_START);
- }
-
- @Override
- public Reader reader() throws IOException {
- return new Reader(in.clone());
- }
-
- @Override
- public void close() throws IOException {
- in.close();
- }
-
- /**
- * Just reads a vInt directly from the file.
- */
- public static class Reader extends IntIndexInput.Reader {
- // clone:
- private final IndexInput in;
-
- public Reader(IndexInput in) {
- this.in = in;
- }
-
- /** Reads next single int */
- @Override
- public int next() throws IOException {
- //System.out.println("msii.next() fp=" + in.getFilePointer() + " vs " + in.length());
- return in.readVInt();
- }
- }
-
- class MockSingleIntIndexInputIndex extends IntIndexInput.Index {
- private long fp;
-
- @Override
- public void read(DataInput indexIn, boolean absolute)
- throws IOException {
- if (absolute) {
- fp = indexIn.readVLong();
- } else {
- fp += indexIn.readVLong();
- }
- }
-
- @Override
- public void copyFrom(IntIndexInput.Index other) {
- fp = ((MockSingleIntIndexInputIndex) other).fp;
- }
-
- @Override
- public void seek(IntIndexInput.Reader other) throws IOException {
- ((Reader) other).in.seek(fp);
- }
-
- @Override
- public String toString() {
- return Long.toString(fp);
- }
-
- @Override
- public Index clone() {
- MockSingleIntIndexInputIndex other = new MockSingleIntIndexInputIndex();
- other.fp = fp;
- return other;
- }
- }
-
- @Override
- public Index index() {
- return new MockSingleIntIndexInputIndex();
- }
-}
-
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntIndexOutput.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntIndexOutput.java
deleted file mode 100644
index c8aa4172e7f..00000000000
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntIndexOutput.java
+++ /dev/null
@@ -1,105 +0,0 @@
-package org.apache.lucene.codecs.mocksep;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.sep.IntIndexOutput;
-
-import java.io.IOException;
-
-/** Writes ints directly to the file (not in blocks) as
- * vInt.
- *
- * @lucene.experimental
-*/
-public class MockSingleIntIndexOutput extends IntIndexOutput {
- private final IndexOutput out;
- final static String CODEC = "SINGLE_INTS";
- final static int VERSION_START = 0;
- final static int VERSION_CURRENT = VERSION_START;
-
- public MockSingleIntIndexOutput(Directory dir, String fileName, IOContext context) throws IOException {
- out = dir.createOutput(fileName, context);
- boolean success = false;
- try {
- CodecUtil.writeHeader(out, CODEC, VERSION_CURRENT);
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(out);
- }
- }
- }
-
- /** Write an int to the primary file */
- @Override
- public void write(int v) throws IOException {
- out.writeVInt(v);
- }
-
- @Override
- public Index index() {
- return new MockSingleIntIndexOutputIndex();
- }
-
- @Override
- public void close() throws IOException {
- out.close();
- }
-
- @Override
- public String toString() {
- return "MockSingleIntIndexOutput fp=" + out.getFilePointer();
- }
-
- private class MockSingleIntIndexOutputIndex extends IntIndexOutput.Index {
- long fp;
- long lastFP;
- @Override
- public void mark() {
- fp = out.getFilePointer();
- }
- @Override
- public void copyFrom(IntIndexOutput.Index other, boolean copyLast) {
- fp = ((MockSingleIntIndexOutputIndex) other).fp;
- if (copyLast) {
- lastFP = ((MockSingleIntIndexOutputIndex) other).fp;
- }
- }
- @Override
- public void write(DataOutput indexOut, boolean absolute)
- throws IOException {
- if (absolute) {
- indexOut.writeVLong(fp);
- } else {
- indexOut.writeVLong(fp - lastFP);
- }
- lastFP = fp;
- }
-
- @Override
- public String toString() {
- return Long.toString(fp);
- }
- }
-}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/package.html b/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/package.html
deleted file mode 100644
index c699ac181e6..00000000000
--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/package.html
+++ /dev/null
@@ -1,25 +0,0 @@
-
-
-
-
- = 0 && allowPayloads;
newFieldInfoArray[fieldUpto] = new FieldInfo(oldFieldInfo.name,
@@ -1232,7 +1224,7 @@ public abstract class BasePostingsFormatTestCase extends LuceneTestCase {
fieldsProducer.close();
dir.close();
- TestUtil.rmDir(path);
+ TestUtil.rm(path);
}
public void testDocsOnly() throws Exception {
@@ -1281,7 +1273,7 @@ public abstract class BasePostingsFormatTestCase extends LuceneTestCase {
fieldsProducer = null;
dir.close();
- TestUtil.rmDir(path);
+ TestUtil.rm(path);
}
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
index a84a253c7dc..04fea7bd2df 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -351,7 +352,7 @@ public abstract class BaseStoredFieldsFormatTestCase extends LuceneTestCase {
ft.freeze();
final String string = TestUtil.randomSimpleString(random(), 50);
- final byte[] bytes = string.getBytes("UTF-8");
+ final byte[] bytes = string.getBytes(StandardCharsets.UTF_8);
final long l = random().nextBoolean() ? random().nextInt(42) : random().nextLong();
final int i = random().nextBoolean() ? random().nextInt(42) : random().nextInt();
final float f = random().nextFloat();
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
index 8da536dd63a..6494a650d04 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
@@ -46,10 +46,7 @@ import org.apache.lucene.codecs.memory.FSTPostingsFormat;
import org.apache.lucene.codecs.memory.FSTPulsing41PostingsFormat;
import org.apache.lucene.codecs.memory.MemoryDocValuesFormat;
import org.apache.lucene.codecs.memory.MemoryPostingsFormat;
-import org.apache.lucene.codecs.mockintblock.MockFixedIntBlockPostingsFormat;
-import org.apache.lucene.codecs.mockintblock.MockVariableIntBlockPostingsFormat;
import org.apache.lucene.codecs.mockrandom.MockRandomPostingsFormat;
-import org.apache.lucene.codecs.mocksep.MockSepPostingsFormat;
import org.apache.lucene.codecs.nestedpulsing.NestedPulsingPostingsFormat;
import org.apache.lucene.codecs.pulsing.Pulsing41PostingsFormat;
import org.apache.lucene.codecs.simpletext.SimpleTextDocValuesFormat;
@@ -142,9 +139,6 @@ public class RandomCodec extends Lucene46Codec {
//with a choice of concrete PostingsFormats. Maybe useful to have a generic means of marking and dealing
//with such "wrapper" classes?
new TestBloomFilteredLucene41Postings(),
- new MockSepPostingsFormat(),
- new MockFixedIntBlockPostingsFormat(TestUtil.nextInt(random, 1, 2000)),
- new MockVariableIntBlockPostingsFormat( TestUtil.nextInt(random, 1, 127)),
new MockRandomPostingsFormat(random),
new NestedPulsingPostingsFormat(),
new Lucene41WithOrds(TestUtil.nextInt(random, 1, 1000)),
diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java
index 0989d9e4977..8745bb5ed58 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java
@@ -165,6 +165,11 @@ public class MockIndexOutputWrapper extends IndexOutput {
dir.maybeThrowDeterministicException();
}
+ @Override
+ public long getChecksum() throws IOException {
+ return delegate.getChecksum();
+ }
+
@Override
public String toString() {
return "MockIndexOutputWrapper(" + delegate + ")";
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/FailureMarker.java b/lucene/test-framework/src/java/org/apache/lucene/util/FailureMarker.java
new file mode 100644
index 00000000000..7487f87aaf7
--- /dev/null
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/FailureMarker.java
@@ -0,0 +1,48 @@
+package org.apache.lucene.util;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.runner.notification.Failure;
+import org.junit.runner.notification.RunListener;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A {@link RunListener} that detects suite/ test failures. We need it because failures
+ * due to thread leaks happen outside of any rule contexts.
+ */
+public class FailureMarker extends RunListener {
+ static final AtomicInteger failures = new AtomicInteger();
+
+ @Override
+ public void testFailure(Failure failure) throws Exception {
+ failures.incrementAndGet();
+ }
+
+ public static boolean hadFailures() {
+ return failures.get() > 0;
+ }
+
+ static int getFailures() {
+ return failures.get();
+ }
+
+ public static void resetFailures() {
+ failures.set(0);
+ }
+}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java b/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java
index 33d9d1d6917..4cc95bb0f3d 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java
@@ -29,6 +29,7 @@ import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.GZIPInputStream;
@@ -133,7 +134,7 @@ public class LineFileDocs implements Closeable {
} while (b >= 0 && b != 13 && b != 10);
}
- CharsetDecoder decoder = IOUtils.CHARSET_UTF_8.newDecoder()
+ CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder()
.onMalformedInput(CodingErrorAction.REPORT)
.onUnmappableCharacter(CodingErrorAction.REPORT);
reader = new BufferedReader(new InputStreamReader(is, decoder), BUFFER_SIZE);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
index 439238e70ee..05d51c23224 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
@@ -126,7 +126,8 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAs
JUnit4MethodProvider.class
})
@Listeners({
- RunListenerPrintReproduceInfo.class
+ RunListenerPrintReproduceInfo.class,
+ FailureMarker.class
})
@SeedDecorators({MixWithSuiteName.class}) // See LUCENE-3995 for rationale.
@ThreadLeakScope(Scope.SUITE)
@@ -314,14 +315,6 @@ public abstract class LuceneTestCase extends Assert {
CORE_DIRECTORIES.add("RAMDirectory");
};
- protected static final Set doesntSupportOffsets = new HashSet<>(Arrays.asList(
- "Lucene3x",
- "MockFixedIntBlock",
- "MockVariableIntBlock",
- "MockSep",
- "MockRandom"
- ));
-
// -----------------------------------------------------------------
// Fields initialized in class or instance rules.
// -----------------------------------------------------------------
@@ -353,8 +346,7 @@ public abstract class LuceneTestCase extends Assert {
/**
* Suite failure marker (any error in the test or suite scope).
*/
- public final static TestRuleMarkFailure suiteFailureMarker =
- new TestRuleMarkFailure();
+ public static TestRuleMarkFailure suiteFailureMarker;
/**
* Ignore tests after hitting a designated number of initial failures. This
@@ -419,7 +411,7 @@ public abstract class LuceneTestCase extends Assert {
public static TestRule classRules = RuleChain
.outerRule(new TestRuleIgnoreTestSuites())
.around(ignoreAfterMaxFailures)
- .around(suiteFailureMarker)
+ .around(suiteFailureMarker = new TestRuleMarkFailure())
.around(new TestRuleAssertionsRequired())
.around(new StaticFieldsInvariantRule(STATIC_LEAK_THRESHOLD, true) {
@Override
@@ -800,6 +792,7 @@ public abstract class LuceneTestCase extends Assert {
}
c.setUseCompoundFile(r.nextBoolean());
c.setReaderPooling(r.nextBoolean());
+ c.setCheckIntegrityAtMerge(r.nextBoolean());
return c;
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIgnoreAfterMaxFailures.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIgnoreAfterMaxFailures.java
index 66667142808..f3296f37825 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIgnoreAfterMaxFailures.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleIgnoreAfterMaxFailures.java
@@ -40,11 +40,6 @@ public final class TestRuleIgnoreAfterMaxFailures implements TestRule {
* Maximum failures. Package scope for tests.
*/
int maxFailures;
-
- /**
- * Current count of failures. Package scope for tests.
- */
- int failuresSoFar;
/**
* @param maxFailures
@@ -61,19 +56,13 @@ public final class TestRuleIgnoreAfterMaxFailures implements TestRule {
return new Statement() {
@Override
public void evaluate() throws Throwable {
+ int failuresSoFar = FailureMarker.getFailures();
if (failuresSoFar >= maxFailures) {
RandomizedTest.assumeTrue("Ignored, failures limit reached (" +
failuresSoFar + " >= " + maxFailures + ").", false);
}
- try {
- s.evaluate();
- } catch (Throwable t) {
- if (!TestRuleMarkFailure.isAssumption(t)) {
- failuresSoFar++;
- }
- throw t;
- }
+ s.evaluate();
}
};
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
index b9cc69a4e1a..6ef10cf99ac 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
@@ -103,42 +103,38 @@ public final class TestUtil {
private static final int GET_TEMP_DIR_RETRY_THRESHOLD = 1000;
/**
- * Deletes a directory and everything underneath it.
+ * Deletes a file or a directory (and everything underneath it).
*/
- public static void rmDir(File dir) throws IOException {
- if (dir.exists()) {
- if (dir.isFile() && !dir.delete()) {
- throw new IOException("could not delete " + dir);
+ public static void rm(File location) throws IOException {
+ if (!location.exists()) {
+ return;
+ }
+
+ if (location.isDirectory()) {
+ for (File f : location.listFiles()) {
+ rm(f);
}
- for (File f : dir.listFiles()) {
- if (f.isDirectory()) {
- rmDir(f);
- } else {
- if (!f.delete()) {
- throw new IOException("could not delete " + f);
- }
- }
- }
- if (!dir.delete()) {
- throw new IOException("could not delete " + dir);
+ } else {
+ if (!location.delete()) {
+ throw new IOException("Could not delete: " + location.getAbsolutePath());
}
}
+
+ assert !location.exists();
}
/**
- * Convenience method: Unzip zipName + ".zip" under destDir, removing destDir first
+ * Convenience method: Unzip zipName + ".zip" into destDir, cleaning up
+ * destDir first.
*/
public static void unzip(File zipName, File destDir) throws IOException {
-
- ZipFile zipFile = new ZipFile(zipName);
-
- Enumeration entries = zipFile.entries();
-
- rmDir(destDir);
-
+ rm(destDir);
destDir.mkdir();
LuceneTestCase.closeAfterSuite(new CloseableFile(destDir, LuceneTestCase.suiteFailureMarker));
+ ZipFile zipFile = new ZipFile(zipName);
+ Enumeration entries = zipFile.entries();
+
while (entries.hasMoreElements()) {
ZipEntry entry = entries.nextElement();
@@ -189,15 +185,15 @@ public final class TestUtil {
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
CheckIndex checker = new CheckIndex(dir);
checker.setCrossCheckTermVectors(crossCheckTermVectors);
- checker.setInfoStream(new PrintStream(bos, false, "UTF-8"), false);
+ checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false);
CheckIndex.Status indexStatus = checker.checkIndex(null);
if (indexStatus == null || indexStatus.clean == false) {
System.out.println("CheckIndex failed");
- System.out.println(bos.toString("UTF-8"));
+ System.out.println(bos.toString(IOUtils.UTF_8));
throw new RuntimeException("CheckIndex failed");
} else {
if (LuceneTestCase.INFOSTREAM) {
- System.out.println(bos.toString("UTF-8"));
+ System.out.println(bos.toString(IOUtils.UTF_8));
}
return indexStatus;
}
@@ -213,8 +209,9 @@ public final class TestUtil {
public static void checkReader(AtomicReader reader, boolean crossCheckTermVectors) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
- PrintStream infoStream = new PrintStream(bos, false, "UTF-8");
+ PrintStream infoStream = new PrintStream(bos, false, IOUtils.UTF_8);
+ reader.checkIntegrity();
FieldNormStatus fieldNormStatus = CheckIndex.testFieldNorms(reader, infoStream);
TermIndexStatus termIndexStatus = CheckIndex.testPostings(reader, infoStream);
StoredFieldStatus storedFieldStatus = CheckIndex.testStoredFields(reader, infoStream);
@@ -227,11 +224,11 @@ public final class TestUtil {
termVectorStatus.error != null ||
docValuesStatus.error != null) {
System.out.println("CheckReader failed");
- System.out.println(bos.toString("UTF-8"));
+ System.out.println(bos.toString(IOUtils.UTF_8));
throw new RuntimeException("CheckReader failed");
} else {
if (LuceneTestCase.INFOSTREAM) {
- System.out.println(bos.toString("UTF-8"));
+ System.out.println(bos.toString(IOUtils.UTF_8));
}
}
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java b/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java
index 8ba2538eb21..169c5498201 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java
@@ -145,4 +145,9 @@ public class ThrottledIndexOutput extends IndexOutput {
public void copyBytes(DataInput input, long numBytes) throws IOException {
delegate.copyBytes(input, numBytes);
}
+
+ @Override
+ public long getChecksum() throws IOException {
+ return delegate.getChecksum();
+ }
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java b/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java
index bb72ab0dd64..3384e9ee7a4 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java
@@ -21,6 +21,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -328,7 +329,7 @@ public class FSTTester {
}
if (LuceneTestCase.VERBOSE && pairs.size() <= 20 && fst != null) {
- Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"), "UTF-8");
+ Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"), StandardCharsets.UTF_8);
Util.toDot(fst, w, false, false);
w.close();
System.out.println("SAVED out.dot");
diff --git a/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
index 59d0dd3f33f..f85f32d7550 100644
--- a/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
+++ b/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
@@ -13,10 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.codecs.mockintblock.MockFixedIntBlockPostingsFormat
-org.apache.lucene.codecs.mockintblock.MockVariableIntBlockPostingsFormat
org.apache.lucene.codecs.mockrandom.MockRandomPostingsFormat
-org.apache.lucene.codecs.mocksep.MockSepPostingsFormat
org.apache.lucene.codecs.nestedpulsing.NestedPulsingPostingsFormat
org.apache.lucene.codecs.ramonly.RAMOnlyPostingsFormat
org.apache.lucene.codecs.lucene41ords.Lucene41WithOrds
diff --git a/lucene/tools/src/java/org/apache/lucene/dependencies/GetMavenDependenciesTask.java b/lucene/tools/src/java/org/apache/lucene/dependencies/GetMavenDependenciesTask.java
index 33402d03414..e7640ced9f2 100644
--- a/lucene/tools/src/java/org/apache/lucene/dependencies/GetMavenDependenciesTask.java
+++ b/lucene/tools/src/java/org/apache/lucene/dependencies/GetMavenDependenciesTask.java
@@ -38,9 +38,9 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Reader;
-import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -58,6 +58,7 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
@@ -91,17 +92,19 @@ public class GetMavenDependenciesTask extends Task {
private static final Properties allProperties = new Properties();
private static final Set modulesWithSeparateCompileAndTestPOMs = new HashSet<>();
- private static final Set optionalExternalDependencies = new HashSet<>();
+ private static final Set globalOptionalExternalDependencies = new HashSet<>();
+ private static final Map> perModuleOptionalExternalDependencies = new HashMap<>();
static {
// Add modules here that have split compile and test POMs
// - they need compile-scope deps to also be test-scope deps.
modulesWithSeparateCompileAndTestPOMs.addAll
(Arrays.asList("lucene-core", "lucene-codecs", "solr-core", "solr-solrj"));
- // Add external dependencies here that should be optional (i.e., not invoke Maven's transitive dep mechanism).
+ // Add external dependencies here that should be optional for all modules
+ // (i.e., not invoke Maven's transitive dependency mechanism).
// Format is "groupId:artifactId"
- optionalExternalDependencies.addAll(Arrays.asList
- ("org.slf4j:jcl-over-slf4j", "org.slf4j:jul-to-slf4j", "org.slf4j:slf4j-api", "org.slf4j:slf4j-log4j12"));
+ globalOptionalExternalDependencies.addAll(Arrays.asList
+ ("org.slf4j:jcl-over-slf4j", "org.slf4j:jul-to-slf4j", "org.slf4j:slf4j-log4j12"));
}
private final XPath xpath = XPathFactory.newInstance().newXPath();
@@ -151,7 +154,7 @@ public class GetMavenDependenciesTask extends Task {
}
public void setVerbose(boolean verbose) {
- verboseLevel = (verbose ? Project.MSG_INFO : Project.MSG_VERBOSE);
+ verboseLevel = (verbose ? Project.MSG_VERBOSE : Project.MSG_INFO);
}
public void setCentralizedVersionsFile(File file) {
@@ -200,12 +203,10 @@ public class GetMavenDependenciesTask extends Task {
Writer writer = null;
try {
FileOutputStream outputStream = new FileOutputStream(mavenDependenciesFiltersFile);
- writer = new OutputStreamWriter(outputStream, "ISO-8859-1");
+ writer = new OutputStreamWriter(outputStream, StandardCharsets.ISO_8859_1);
allProperties.store(writer, null);
} catch (FileNotFoundException e) {
throw new BuildException("Can't find file: '" + mavenDependenciesFiltersFile.getPath() + "'", e);
- } catch (UnsupportedEncodingException e) {
- throw new BuildException(e);
} catch (IOException e) {
throw new BuildException("Exception writing out '" + mavenDependenciesFiltersFile.getPath() + "'", e);
} finally {
@@ -242,7 +243,7 @@ public class GetMavenDependenciesTask extends Task {
} catch (BuildException e) {
throw e;
} catch (Exception e) {
- throw new BuildException("Exception reading file " + ivyXmlFile.getPath(), e);
+ throw new BuildException("Exception reading file " + ivyXmlFile.getPath() + ": " + e, e);
}
}
addSharedExternalDependencies();
@@ -258,10 +259,10 @@ public class GetMavenDependenciesTask extends Task {
// Delay adding shared compile-scope dependencies until after all have been processed,
// so dependency sharing is limited to a depth of one.
Map> sharedDependencies = new HashMap<>();
- for (String artifactId : interModuleExternalCompileScopeDependencies.keySet()) {
+ for (String module : interModuleExternalCompileScopeDependencies.keySet()) {
TreeSet deps = new TreeSet<>();
- sharedDependencies.put(artifactId, deps);
- Set moduleDependencies = interModuleExternalCompileScopeDependencies.get(artifactId);
+ sharedDependencies.put(module, deps);
+ Set moduleDependencies = interModuleExternalCompileScopeDependencies.get(module);
if (null != moduleDependencies) {
for (String otherArtifactId : moduleDependencies) {
SortedSet otherExtDeps = allExternalDependencies.get(otherArtifactId);
@@ -275,13 +276,13 @@ public class GetMavenDependenciesTask extends Task {
}
}
}
- for (String artifactId : interModuleExternalTestScopeDependencies.keySet()) {
- SortedSet deps = sharedDependencies.get(artifactId);
+ for (String module : interModuleExternalTestScopeDependencies.keySet()) {
+ SortedSet deps = sharedDependencies.get(module);
if (null == deps) {
deps = new TreeSet<>();
- sharedDependencies.put(artifactId, deps);
+ sharedDependencies.put(module, deps);
}
- Set moduleDependencies = interModuleExternalTestScopeDependencies.get(artifactId);
+ Set moduleDependencies = interModuleExternalTestScopeDependencies.get(module);
if (null != moduleDependencies) {
for (String otherArtifactId : moduleDependencies) {
int testScopePos = otherArtifactId.indexOf(":test");
@@ -295,8 +296,8 @@ public class GetMavenDependenciesTask extends Task {
for (ExternalDependency otherDep : otherExtDeps) {
if (otherDep.isTestDependency == isTestScope) {
if ( ! deps.contains(otherDep)
- && ( null == allExternalDependencies.get(artifactId)
- || ! allExternalDependencies.get(artifactId).contains(otherDep))) {
+ && ( null == allExternalDependencies.get(module)
+ || ! allExternalDependencies.get(module).contains(otherDep))) {
// Add test-scope clone only if it's not already a compile-scope dependency.
ExternalDependency otherDepTestScope = new ExternalDependency
(otherDep.groupId, otherDep.artifactId, otherDep.classifier, true, otherDep.isOptional);
@@ -308,13 +309,21 @@ public class GetMavenDependenciesTask extends Task {
}
}
}
- for (String artifactId : sharedDependencies.keySet()) {
- SortedSet deps = allExternalDependencies.get(artifactId);
+ for (String module : sharedDependencies.keySet()) {
+ SortedSet deps = allExternalDependencies.get(module);
if (null == deps) {
deps = new TreeSet<>();
- allExternalDependencies.put(artifactId, deps);
+ allExternalDependencies.put(module, deps);
+ }
+ for (ExternalDependency dep : sharedDependencies.get(module)) {
+ String dependencyCoordinate = dep.groupId + ":" + dep.artifactId;
+ if (globalOptionalExternalDependencies.contains(dependencyCoordinate)
+ || (perModuleOptionalExternalDependencies.containsKey(module)
+ && perModuleOptionalExternalDependencies.get(module).contains(dependencyCoordinate))) {
+ dep = new ExternalDependency(dep.groupId, dep.artifactId, dep.classifier, dep.isTestDependency, true);
+ }
+ deps.add(dep);
}
- deps.addAll(sharedDependencies.get(artifactId));
}
}
@@ -615,7 +624,7 @@ public class GetMavenDependenciesTask extends Task {
// Pattern.compile("(lucene|solr)/build/(.*)/classes/java");
String artifact = matcher.group(2);
artifact = artifact.replace('/', '-');
- artifact = artifact.replace("analysis-", "analyzers-");
+ artifact = artifact.replace("(? deps = allExternalDependencies.get(module);
if (null == deps) {
deps = new TreeSet<>();
@@ -808,7 +819,7 @@ public class GetMavenDependenciesTask extends Task {
throw new BuildException("Properties file does not exist: " + file.getPath());
}
// Properties files are encoded as Latin-1
- final Reader reader = new InputStreamReader(stream, Charset.forName("ISO-8859-1"));
+ final Reader reader = new InputStreamReader(stream, StandardCharsets.ISO_8859_1);
final Properties properties = new Properties();
try {
properties.load(reader);
diff --git a/lucene/tools/src/java/org/apache/lucene/validation/LibVersionsCheckTask.java b/lucene/tools/src/java/org/apache/lucene/validation/LibVersionsCheckTask.java
index 7741fb6e971..9777df9a965 100644
--- a/lucene/tools/src/java/org/apache/lucene/validation/LibVersionsCheckTask.java
+++ b/lucene/tools/src/java/org/apache/lucene/validation/LibVersionsCheckTask.java
@@ -33,6 +33,7 @@ import org.xml.sax.helpers.DefaultHandler;
import org.xml.sax.helpers.XMLReaderFactory;
import javax.xml.parsers.ParserConfigurationException;
+
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
@@ -44,6 +45,7 @@ import java.io.Reader;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Locale;
@@ -188,7 +190,7 @@ public class LibVersionsCheckTask extends Task {
+ centralizedVersionsFile.getPath());
}
// Properties files are encoded as Latin-1
- final Reader reader = new InputStreamReader(stream, Charset.forName("ISO-8859-1"));
+ final Reader reader = new InputStreamReader(stream, StandardCharsets.ISO_8859_1);
final BufferedReader bufferedReader = new BufferedReader(reader);
String line = null;
diff --git a/lucene/tools/src/java/org/apache/lucene/validation/LicenseCheckTask.java b/lucene/tools/src/java/org/apache/lucene/validation/LicenseCheckTask.java
index 7ee4a6a598e..c3a8d0ccefd 100644
--- a/lucene/tools/src/java/org/apache/lucene/validation/LicenseCheckTask.java
+++ b/lucene/tools/src/java/org/apache/lucene/validation/LicenseCheckTask.java
@@ -22,14 +22,13 @@ import java.io.FileInputStream;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.IOException;
-
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
-
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
@@ -302,7 +301,7 @@ outer:
BufferedReader reader = null;
try {
reader = new BufferedReader(new InputStreamReader
- (new FileInputStream(f), "UTF-8"));
+ (new FileInputStream(f), StandardCharsets.UTF_8));
try {
String checksum = reader.readLine();
if (null == checksum || 0 == checksum.length()) {
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 58b9b04e3a5..fb3abaa5864 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -146,9 +146,6 @@ New Features
improve logging and force refresh cluster state every 15 seconds.
(Timothy Potter via shalin)
- * SOLR-5228: Don't require or be inside of -- or
- that be inside of . (Erick Erickson)
-
* SOLR-5749: A new Overseer status collection API exposes overseer queue sizes, timing
statistics, success and error counts and last N failures per operation. (shalin)
@@ -159,6 +156,21 @@ New Features
* SOLR-4478: Allow cores to use configuration from a configsets directory
outside their instance directory. (Alan Woodward, Erick Erickson)
+* SOLR-5466: A new List collections and cluster status API which clients can use
+ to read collection and shard information instead of reading data directly from ZooKeeper.
+ (Dave Seltzer, Varun Thacker, Vitaliy Zhovtyuk, Erick Erickson, shalin)
+
+* SOLR-5795: New DocExpirationUpdateProcessorFactory supports computing an expiration
+ date for documents from the "TTL" expression, as well as automatically deleting expired
+ documents on a periodic basis. (hossman)
+
+* SOLR-5829: Allow ExpandComponent to accept query and filter query parameters
+ (Joel Bernstein)
+
+* SOLR-5654: Create a synonym filter factory that is (re)configurable, and
+ capable of reporting its configuration, via REST API.
+ (Tim Potter via Steve Rowe)
+
Bug Fixes
----------------------
@@ -167,6 +179,20 @@ Bug Fixes
* SOLR-5893: On restarting overseer designate , move itself to front of the queue (Noble Paul)
+* SOLR-5915: Attempts to specify the parserImpl for
+ solr.PreAnalyzedField fieldtype failed. (Mike McCandless)
+
+* SOLR-5943: SolrCmdDistributor does not distribute the openSearcher parameter.
+ (ludovic Boutros via shalin)
+
+* SOLR-5951: Fixed SolrDispatchFilter to throw useful exception on startup if
+ SLF4j logging jars are missing. (Uwe Schindler, Hossman, Shawn Heisey)
+
+* SOLR-5950: Maven config: make the org.slf4j:slf4j-api dependency transitive
+ (i.e., not optional) in all modules in which it's a dependency, including
+ solrj, except for the WAR, where it will remain optional.
+ (Uwe Schindler, Steve Rowe)
+
Optimizations
----------------------
* SOLR-1880: Distributed Search skips GET_FIELDS stage if EXECUTE_QUERY
@@ -243,6 +269,12 @@ Other Changes
* SOLR-5914: Cleanup and fix Solr's test cleanup code.
(Mark Miller, Uwe Schindler)
+* SOLR-5934: LBHttpSolrServer exception handling improvement and small test
+ improvements. (Gregory Chanan via Mark Miller)
+
+* SOLR-5773: CollapsingQParserPlugin should make elevated documents the
+ group head. (David Boychuck, Joel Bernstein)
+
================== 4.7.1 ==================
Versions of Major Components
@@ -344,6 +376,9 @@ Bug Fixes
* SOLR-5906: Collection create API ignores property.instanceDir parameter.
(Varun Thacker, shalin)
+* SOLR-5920: Distributed sort on DateField, BoolField and BCD{Int,Long,Str}Field
+ returns string cast exception (Eric Bus, AJ Lemke, hossman, Steve Rowe)
+
Other Changes
---------------------
diff --git a/solr/common-build.xml b/solr/common-build.xml
index 001488b66fe..f2b168d6faf 100644
--- a/solr/common-build.xml
+++ b/solr/common-build.xml
@@ -445,6 +445,12 @@
+
+
+
+
+
diff --git a/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java b/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
index bed79e6ffb9..b798bc96496 100644
--- a/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
+++ b/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
@@ -43,7 +43,6 @@ import org.apache.lucene.util.Version;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.util.Base64;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
@@ -303,20 +302,11 @@ public class ICUCollationField extends FieldType {
@Override
public Object marshalSortValue(Object value) {
- if (null == value) {
- return null;
- }
- final BytesRef val = (BytesRef)value;
- return Base64.byteArrayToBase64(val.bytes, val.offset, val.length);
+ return marshalBase64SortValue(value);
}
@Override
public Object unmarshalSortValue(Object value) {
- if (null == value) {
- return null;
- }
- final String val = (String)value;
- final byte[] bytes = Base64.base64ToByteArray(val);
- return new BytesRef(bytes);
+ return unmarshalBase64SortValue(value);
}
}
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java
index 3ba608c4990..20b34eedd16 100644
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java
+++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java
@@ -18,10 +18,12 @@ package org.apache.solr.handler.dataimport;
import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
+import java.nio.charset.StandardCharsets;
import java.sql.Blob;
import java.sql.Clob;
import java.sql.SQLException;
@@ -106,7 +108,7 @@ public class FieldReaderDataSource extends DataSource {
private Reader getReader(Blob blob)
throws SQLException, UnsupportedEncodingException {
if (encoding == null) {
- return (new InputStreamReader(blob.getBinaryStream(), "UTF-8"));
+ return (new InputStreamReader(blob.getBinaryStream(), StandardCharsets.UTF_8));
} else {
return (new InputStreamReader(blob.getBinaryStream(), encoding));
}
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java
index 8001935e889..5f5136bcb3e 100644
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java
+++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java
@@ -17,10 +17,12 @@
package org.apache.solr.handler.dataimport;
import java.io.*;
+import java.nio.charset.StandardCharsets;
import java.util.Properties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
@@ -138,7 +140,7 @@ public class FileDataSource extends DataSource {
protected Reader openStream(File file) throws FileNotFoundException,
UnsupportedEncodingException {
if (encoding == null) {
- return new InputStreamReader(new FileInputStream(file), "UTF-8");
+ return new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8);
} else {
return new InputStreamReader(new FileInputStream(file), encoding);
}
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java
index 73e6fb1c6fb..7857ab90cf7 100644
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java
+++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java
@@ -26,6 +26,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
@@ -192,7 +193,7 @@ public class SimplePropertiesWriter extends DIHProperties {
filePath += File.separator;
}
filePath += filename;
- propOutput = new OutputStreamWriter(new FileOutputStream(filePath), IOUtils.CHARSET_UTF_8);
+ propOutput = new OutputStreamWriter(new FileOutputStream(filePath), StandardCharsets.UTF_8);
existingProps.store(propOutput, null);
log.info("Wrote last indexed time to " + filename);
} catch (Exception e) {
@@ -214,7 +215,7 @@ public class SimplePropertiesWriter extends DIHProperties {
}
filePath += filename;
propInput = new FileInputStream(filePath);
- props.load(new InputStreamReader(propInput, IOUtils.CHARSET_UTF_8));
+ props.load(new InputStreamReader(propInput, StandardCharsets.UTF_8));
log.info("Read " + filename);
} catch (Exception e) {
log.warn("Unable to read: " + filename);
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java
index a87745e1433..61cbaa3e20b 100644
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java
+++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java
@@ -28,6 +28,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
+import java.nio.charset.StandardCharsets;
/**
* Writes documents to SOLR.
@@ -147,7 +148,7 @@ public class SolrWriter extends DIHWriterBase implements DIHWriter {
}
}
- return new String(baos.toByteArray(), "UTF-8");
+ return new String(baos.toByteArray(), StandardCharsets.UTF_8);
}
static String getDocCount() {
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/URLDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/URLDataSource.java
index f462466a557..b7609ade6ea 100644
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/URLDataSource.java
+++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/URLDataSource.java
@@ -24,6 +24,7 @@ import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URL;
import java.net.URLConnection;
+import java.nio.charset.StandardCharsets;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -140,7 +141,7 @@ public class URLDataSource extends DataSource {
public static final String BASE_URL = "baseUrl";
- public static final String UTF_8 = "UTF-8";
+ public static final String UTF_8 = StandardCharsets.UTF_8.name();
public static final String CONNECTION_TIMEOUT_FIELD_NAME = "connectionTimeout";
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/UrlEvaluator.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/UrlEvaluator.java
index 2d8c1990bc4..78f95234e25 100644
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/UrlEvaluator.java
+++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/UrlEvaluator.java
@@ -4,6 +4,7 @@ import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVE
import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.util.List;
/*
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java
index d839f277605..36a05cdd831 100644
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java
+++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java
@@ -18,10 +18,10 @@ package org.apache.solr.handler.dataimport;
import java.io.StringReader;
import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.Properties;
-import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.zookeeper.KeeperException.NodeExistsException;
import org.slf4j.Logger;
@@ -67,7 +67,7 @@ public class ZKPropertiesWriter extends SimplePropertiesWriter {
StringWriter output = new StringWriter();
try {
existing.store(output, null);
- byte[] bytes = output.toString().getBytes(IOUtils.CHARSET_UTF_8);
+ byte[] bytes = output.toString().getBytes(StandardCharsets.UTF_8);
if (!zkClient.exists(path, false)) {
try {
zkClient.makePath(path, false);
@@ -90,7 +90,7 @@ public class ZKPropertiesWriter extends SimplePropertiesWriter {
try {
byte[] data = zkClient.getData(path, null, null, false);
if (data != null) {
- props.load(new StringReader(new String(data, "UTF-8")));
+ props.load(new StringReader(new String(data, StandardCharsets.UTF_8)));
}
} catch (Exception e) {
log.warn(
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
index 7b8d632ad54..12c2e7f4c22 100644
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
+++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestBuiltInEvaluators.java
@@ -20,6 +20,7 @@ import org.junit.Before;
import org.junit.Test;
import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.*;
@@ -30,7 +31,7 @@ import java.util.*;
* @since solr 1.3
*/
public class TestBuiltInEvaluators extends AbstractDataImportHandlerTestCase {
- private static final String ENCODING = "UTF-8";
+ private static final String ENCODING = StandardCharsets.UTF_8.name();
VariableResolver resolver;
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java
index f946af943ed..faa7dcfac29 100644
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java
+++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java
@@ -26,6 +26,8 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
+import java.nio.charset.StandardCharsets;
+
/**
*
* Test for DocBuilder using the test harness
@@ -243,14 +245,14 @@ public class TestDocBuilder2 extends AbstractDataImportHandlerTestCase {
Map params = createMap("baseDir", tmpdir.getAbsolutePath());
- createFile(tmpdir, "a.xml", "a.xml".getBytes("UTF-8"), true);
- createFile(tmpdir, "b.xml", "b.xml".getBytes("UTF-8"), true);
- createFile(tmpdir, "c.props", "c.props".getBytes("UTF-8"), true);
+ createFile(tmpdir, "a.xml", "a.xml".getBytes(StandardCharsets.UTF_8), true);
+ createFile(tmpdir, "b.xml", "b.xml".getBytes(StandardCharsets.UTF_8), true);
+ createFile(tmpdir, "c.props", "c.props".getBytes(StandardCharsets.UTF_8), true);
runFullImport(dataConfigFileList, params);
assertQ(req("*:*"), "//*[@numFound='3']");
// Add a new file after a full index is done
- createFile(tmpdir, "t.xml", "t.xml".getBytes("UTF-8"), false);
+ createFile(tmpdir, "t.xml", "t.xml".getBytes(StandardCharsets.UTF_8), false);
runFullImport(dataConfigFileList, params);
// we should find only 1 because by default clean=true is passed
// and this particular import should find only one file t.xml
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
index 9ab861d7657..1fac521c9d4 100644
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
+++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
@@ -23,6 +23,7 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.*;
@@ -41,9 +42,9 @@ public class TestFileListEntityProcessor extends AbstractDataImportHandlerTestCa
public void testSimple() throws IOException {
File tmpdir = createTempDir();
- createFile(tmpdir, "a.xml", "a.xml".getBytes("UTF-8"), false);
- createFile(tmpdir, "b.xml", "b.xml".getBytes("UTF-8"), false);
- createFile(tmpdir, "c.props", "c.props".getBytes("UTF-8"), false);
+ createFile(tmpdir, "a.xml", "a.xml".getBytes(StandardCharsets.UTF_8), false);
+ createFile(tmpdir, "b.xml", "b.xml".getBytes(StandardCharsets.UTF_8), false);
+ createFile(tmpdir, "c.props", "c.props".getBytes(StandardCharsets.UTF_8), false);
Map attrs = createMap(
FileListEntityProcessor.FILE_NAME, "xml$",
FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath());
@@ -69,19 +70,19 @@ public class TestFileListEntityProcessor extends AbstractDataImportHandlerTestCa
long minLength = Long.MAX_VALUE;
String smallestFile = "";
- byte[] content = "abcdefgij".getBytes("UTF-8");
+ byte[] content = "abcdefgij".getBytes(StandardCharsets.UTF_8);
createFile(tmpdir, "a.xml", content, false);
if (minLength > content.length) {
minLength = content.length;
smallestFile = "a.xml";
}
- content = "abcdefgij".getBytes("UTF-8");
+ content = "abcdefgij".getBytes(StandardCharsets.UTF_8);
createFile(tmpdir, "b.xml", content, false);
if (minLength > content.length) {
minLength = content.length;
smallestFile = "b.xml";
}
- content = "abc".getBytes("UTF-8");
+ content = "abc".getBytes(StandardCharsets.UTF_8);
createFile(tmpdir, "c.props", content, false);
if (minLength > content.length) {
minLength = content.length;
@@ -135,9 +136,9 @@ public class TestFileListEntityProcessor extends AbstractDataImportHandlerTestCa
public void testNTOT() throws IOException {
File tmpdir = createTempDir();
- createFile(tmpdir, "a.xml", "a.xml".getBytes("UTF-8"), true);
- createFile(tmpdir, "b.xml", "b.xml".getBytes("UTF-8"), true);
- createFile(tmpdir, "c.props", "c.props".getBytes("UTF-8"), true);
+ createFile(tmpdir, "a.xml", "a.xml".getBytes(StandardCharsets.UTF_8), true);
+ createFile(tmpdir, "b.xml", "b.xml".getBytes(StandardCharsets.UTF_8), true);
+ createFile(tmpdir, "c.props", "c.props".getBytes(StandardCharsets.UTF_8), true);
Map attrs = createMap(
FileListEntityProcessor.FILE_NAME, "xml$",
FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
@@ -159,7 +160,7 @@ public class TestFileListEntityProcessor extends AbstractDataImportHandlerTestCa
VariableResolver resolver = new VariableResolver();
String lastMod = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).format(new Date(System.currentTimeMillis() - 50000));
resolver.addNamespace("a", createMap("x", lastMod));
- createFile(tmpdir, "t.xml", "t.xml".getBytes("UTF-8"), false);
+ createFile(tmpdir, "t.xml", "t.xml".getBytes(StandardCharsets.UTF_8), false);
fList = getFiles(resolver, attrs);
assertEquals(1, fList.size());
assertEquals("File name must be t.xml", new File(tmpdir, "t.xml").getAbsolutePath(), fList.get(0));
@@ -170,9 +171,9 @@ public class TestFileListEntityProcessor extends AbstractDataImportHandlerTestCa
File tmpdir = createTempDir();
File childdir = new File(tmpdir + "/child" );
childdir.mkdir();
- createFile(childdir, "a.xml", "a.xml".getBytes("UTF-8"), true);
- createFile(childdir, "b.xml", "b.xml".getBytes("UTF-8"), true);
- createFile(childdir, "c.props", "c.props".getBytes("UTF-8"), true);
+ createFile(childdir, "a.xml", "a.xml".getBytes(StandardCharsets.UTF_8), true);
+ createFile(childdir, "b.xml", "b.xml".getBytes(StandardCharsets.UTF_8), true);
+ createFile(childdir, "c.props", "c.props".getBytes(StandardCharsets.UTF_8), true);
Map attrs = createMap(
FileListEntityProcessor.FILE_NAME, "^.*\\.xml$",
FileListEntityProcessor.BASE_DIR, childdir.getAbsolutePath(),
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListWithLineEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListWithLineEntityProcessor.java
index c07a6cf29a3..5f0c58d0aa8 100644
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListWithLineEntityProcessor.java
+++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListWithLineEntityProcessor.java
@@ -1,6 +1,7 @@
package org.apache.solr.handler.dataimport;
import java.io.File;
+import java.nio.charset.StandardCharsets;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -32,9 +33,9 @@ public class TestFileListWithLineEntityProcessor extends AbstractDataImportHandl
public void test() throws Exception {
File tmpdir = TestUtil.createTempDir(LuceneTestCase.getTestClass().getSimpleName());
- createFile(tmpdir, "a.txt", "a line one\na line two\na line three".getBytes("UTF-8"), false);
- createFile(tmpdir, "b.txt", "b line one\nb line two".getBytes("UTF-8"), false);
- createFile(tmpdir, "c.txt", "c line one\nc line two\nc line three\nc line four".getBytes("UTF-8"), false);
+ createFile(tmpdir, "a.txt", "a line one\na line two\na line three".getBytes(StandardCharsets.UTF_8), false);
+ createFile(tmpdir, "b.txt", "b line one\nb line two".getBytes(StandardCharsets.UTF_8), false);
+ createFile(tmpdir, "c.txt", "c line one\nc line two\nc line three\nc line four".getBytes(StandardCharsets.UTF_8), false);
String config = generateConfig(tmpdir);
LocalSolrQueryRequest request = lrf.makeRequest(
diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
index c370223fdb8..778d1913847 100644
--- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
+++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestXPathEntityProcessor.java
@@ -19,6 +19,7 @@ package org.apache.solr.handler.dataimport;
import java.io.File;
import java.io.Reader;
import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -44,7 +45,7 @@ public class TestXPathEntityProcessor extends AbstractDataImportHandlerTestCase
public void withFieldsAndXpath() throws Exception {
File tmpdir = createTempDir();
- createFile(tmpdir, "x.xsl", xsl.getBytes("UTF-8"), false);
+ createFile(tmpdir, "x.xsl", xsl.getBytes(StandardCharsets.UTF_8), false);
Map entityAttrs = createMap("name", "e", "url", "cd.xml",
XPathEntityProcessor.FOR_EACH, "/catalog/cd");
List fields = new ArrayList();
@@ -332,7 +333,7 @@ public class TestXPathEntityProcessor extends AbstractDataImportHandlerTestCase
@Test
public void withDefaultSolrAndXsl() throws Exception {
File tmpdir = createTempDir();
- AbstractDataImportHandlerTestCase.createFile(tmpdir, "x.xsl", xsl.getBytes("UTF-8"),
+ AbstractDataImportHandlerTestCase.createFile(tmpdir, "x.xsl", xsl.getBytes(StandardCharsets.UTF_8),
false);
Map entityAttrs = createMap("name", "e",
diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/RegexRulesPasswordProvider.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/RegexRulesPasswordProvider.java
index 35b8f3c8c6e..8e30d1a880e 100644
--- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/RegexRulesPasswordProvider.java
+++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/RegexRulesPasswordProvider.java
@@ -20,6 +20,7 @@ package org.apache.solr.handler.extraction;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
import java.util.LinkedHashMap;
import java.util.Map.Entry;
import java.util.regex.Pattern;
@@ -73,7 +74,7 @@ public class RegexRulesPasswordProvider implements PasswordProvider {
*/
public static LinkedHashMap parseRulesFile(InputStream is) {
LinkedHashMap rules = new LinkedHashMap<>();
- BufferedReader br = new BufferedReader(IOUtils.getDecodingReader(is, IOUtils.CHARSET_UTF_8));
+ BufferedReader br = new BufferedReader(IOUtils.getDecodingReader(is, StandardCharsets.UTF_8));
String line;
try {
int linenum = 0;
diff --git a/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessorFactory.java b/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessorFactory.java
index 9a50840f578..3b83ea79306 100644
--- a/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessorFactory.java
+++ b/solr/contrib/langid/src/java/org/apache/solr/update/processor/LangDetectLanguageIdentifierUpdateProcessorFactory.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
@@ -126,10 +127,9 @@ public class LangDetectLanguageIdentifierUpdateProcessorFactory extends
}
loaded = true;
List profileData = new ArrayList<>();
- Charset encoding = Charset.forName("UTF-8");
for (String language : languages) {
InputStream stream = LangDetectLanguageIdentifierUpdateProcessor.class.getResourceAsStream("langdetect-profiles/" + language);
- BufferedReader reader = new BufferedReader(new InputStreamReader(stream, encoding));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8));
profileData.add(new String(IOUtils.toCharArray(reader)));
reader.close();
}
diff --git a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java
index 87e6e39074f..55066ffb12e 100644
--- a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java
+++ b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java
@@ -31,6 +31,7 @@ import java.io.Writer;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLClassLoader;
+import java.nio.charset.StandardCharsets;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Arrays;
@@ -82,7 +83,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.kitesdk.morphline.base.Fields;
-import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.io.ByteStreams;
@@ -924,7 +924,7 @@ public class MapReduceIndexerTool extends Configured implements Tool {
FileSystem fs = fullInputList.getFileSystem(conf);
FSDataOutputStream out = fs.create(fullInputList);
try {
- Writer writer = new BufferedWriter(new OutputStreamWriter(out, "UTF-8"));
+ Writer writer = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8));
for (Path inputFile : inputFiles) {
FileSystem inputFileFs = inputFile.getFileSystem(conf);
@@ -949,7 +949,7 @@ public class MapReduceIndexerTool extends Configured implements Tool {
in = inputList.getFileSystem(conf).open(inputList);
}
try {
- BufferedReader reader = new BufferedReader(new InputStreamReader(in, "UTF-8"));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
String line;
while ((line = reader.readLine()) != null) {
writer.write(line + "\n");
@@ -988,7 +988,7 @@ public class MapReduceIndexerTool extends Configured implements Tool {
private void randomizeFewInputFiles(FileSystem fs, Path outputStep2Dir, Path fullInputList) throws IOException {
List lines = new ArrayList();
- BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(fullInputList), "UTF-8"));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(fullInputList), StandardCharsets.UTF_8));
try {
String line;
while ((line = reader.readLine()) != null) {
@@ -1001,7 +1001,7 @@ public class MapReduceIndexerTool extends Configured implements Tool {
Collections.shuffle(lines, new Random(421439783L)); // constant seed for reproducability
FSDataOutputStream out = fs.create(new Path(outputStep2Dir, FULL_INPUT_LIST));
- Writer writer = new BufferedWriter(new OutputStreamWriter(out, "UTF-8"));
+ Writer writer = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8));
try {
for (String line : lines) {
writer.write(line + "\n");
@@ -1135,7 +1135,7 @@ public class MapReduceIndexerTool extends Configured implements Tool {
* turnaround during trial & debug sessions
*/
private void dryRun(MorphlineMapRunner runner, FileSystem fs, Path fullInputList) throws IOException {
- BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(fullInputList), "UTF-8"));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(fullInputList), StandardCharsets.UTF_8));
try {
String line;
while ((line = reader.readLine()) != null) {
@@ -1154,7 +1154,7 @@ public class MapReduceIndexerTool extends Configured implements Tool {
int numFiles = 0;
FSDataOutputStream out = fs.create(fullInputList);
try {
- Writer writer = new BufferedWriter(new OutputStreamWriter(out, "UTF-8"));
+ Writer writer = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8));
for (FileStatus stat : dirs) {
LOG.debug("Adding path {}", stat.getPath());
Path dir = new Path(stat.getPath(), "data/index");
@@ -1263,7 +1263,7 @@ public class MapReduceIndexerTool extends Configured implements Tool {
byte[] bytes = ByteStreams.toByteArray(in);
in.close();
Preconditions.checkArgument(bytes.length > 0);
- int solrShard = Integer.parseInt(new String(bytes, Charsets.UTF_8));
+ int solrShard = Integer.parseInt(new String(bytes, StandardCharsets.UTF_8));
if (!delete(solrShardNumberFile, false, fs)) {
return false;
}
diff --git a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrOutputFormat.java b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrOutputFormat.java
index 3de00b8d445..9f0498dc455 100644
--- a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrOutputFormat.java
+++ b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrOutputFormat.java
@@ -22,6 +22,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Locale;
@@ -242,7 +243,7 @@ public class SolrOutputFormat extends FileOutputFormat {
ZipEntry ze = new ZipEntry("solr.xml");
zos.putNextEntry(ze);
- zos.write("
LOG.debug("Merging into outputShardNum: " + outputShardNum + " from taskId: " + taskId);
Path shardNumberFile = new Path(workDir.getParent().getParent(), TreeMergeMapper.SOLR_SHARD_NUMBER);
OutputStream out = shardNumberFile.getFileSystem(context.getConfiguration()).create(shardNumberFile);
- Writer writer = new OutputStreamWriter(out, Charsets.UTF_8);
+ Writer writer = new OutputStreamWriter(out, StandardCharsets.UTF_8);
writer.write(String.valueOf(outputShardNum));
writer.flush();
writer.close();
diff --git a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/UnbufferedDataInputInputStream.java b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/UnbufferedDataInputInputStream.java
index 1ad141a4264..8a5eaaf6d9e 100644
--- a/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/UnbufferedDataInputInputStream.java
+++ b/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/UnbufferedDataInputInputStream.java
@@ -22,6 +22,7 @@ import java.io.DataInput;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
public class UnbufferedDataInputInputStream extends org.apache.solr.common.util.DataInputInputStream {
private final DataInputStream in;
@@ -97,7 +98,7 @@ public class UnbufferedDataInputInputStream extends org.apache.solr.common.util.
@Override
public String readLine() throws IOException {
- BufferedReader reader = new BufferedReader(new InputStreamReader(in, "UTF-8"));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
return reader.readLine();
}
diff --git a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MapReduceIndexerToolArgumentParserTest.java b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MapReduceIndexerToolArgumentParserTest.java
index d00b904a9d4..e7b21178daf 100644
--- a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MapReduceIndexerToolArgumentParserTest.java
+++ b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MapReduceIndexerToolArgumentParserTest.java
@@ -20,6 +20,7 @@ import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
@@ -191,7 +192,7 @@ public class MapReduceIndexerToolArgumentParserTest extends SolrTestCaseJ4 {
public void testArgsParserHelp() throws UnsupportedEncodingException {
String[] args = new String[] { "--help" };
assertEquals(new Integer(0), parser.parseArgs(args, conf, opts));
- String helpText = new String(bout.toByteArray(), "UTF-8");
+ String helpText = new String(bout.toByteArray(), StandardCharsets.UTF_8);
assertTrue(helpText.contains("MapReduce batch job driver that "));
assertTrue(helpText.contains("bin/hadoop command"));
assertEquals(0, berr.toByteArray().length);
@@ -458,9 +459,9 @@ public class MapReduceIndexerToolArgumentParserTest extends SolrTestCaseJ4 {
private void assertArgumentParserException(String[] args) throws UnsupportedEncodingException {
assertEquals("should have returned fail code", new Integer(1), parser.parseArgs(args, conf, opts));
- assertEquals("no sys out expected:" + new String(bout.toByteArray(), "UTF-8"), 0, bout.toByteArray().length);
+ assertEquals("no sys out expected:" + new String(bout.toByteArray(), StandardCharsets.UTF_8), 0, bout.toByteArray().length);
String usageText;
- usageText = new String(berr.toByteArray(), "UTF-8");
+ usageText = new String(berr.toByteArray(), StandardCharsets.UTF_8);
assertTrue("should start with usage msg \"usage: hadoop \":" + usageText, usageText.startsWith("usage: hadoop "));
}
diff --git a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java
index 870616f0790..a076851353e 100644
--- a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java
+++ b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java
@@ -22,6 +22,7 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.lang.reflect.Array;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import org.apache.commons.io.FileUtils;
@@ -308,7 +309,7 @@ public class MorphlineBasicMiniMRTest extends SolrTestCaseJ4 {
assertTrue(fs.mkdirs(inDir));
Path INPATH = new Path(inDir, "input.txt");
OutputStream os = fs.create(INPATH);
- Writer wr = new OutputStreamWriter(os, "UTF-8");
+ Writer wr = new OutputStreamWriter(os, StandardCharsets.UTF_8);
wr.write(DATADIR + "/" + inputAvroFile);
wr.close();
diff --git a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java
index c6dcce977bb..8f47c9413a3 100644
--- a/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java
+++ b/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java
@@ -24,6 +24,7 @@ import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.lang.reflect.Array;
import java.net.URI;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -679,7 +680,7 @@ public class MorphlineGoLiveMiniMRTest extends AbstractFullDistribZkTestBase {
Path dataDir, String localFile) throws IOException, UnsupportedEncodingException {
Path INPATH = new Path(inDir, "input.txt");
OutputStream os = fs.create(INPATH);
- Writer wr = new OutputStreamWriter(os, "UTF-8");
+ Writer wr = new OutputStreamWriter(os, StandardCharsets.UTF_8);
wr.write(DATADIR + File.separator + localFile);
wr.close();
diff --git a/solr/contrib/velocity/src/java/org/apache/solr/response/SolrParamResourceLoader.java b/solr/contrib/velocity/src/java/org/apache/solr/response/SolrParamResourceLoader.java
index 54d86ddc1d3..8f033b585ed 100644
--- a/solr/contrib/velocity/src/java/org/apache/solr/response/SolrParamResourceLoader.java
+++ b/solr/contrib/velocity/src/java/org/apache/solr/response/SolrParamResourceLoader.java
@@ -25,7 +25,7 @@ import org.apache.commons.collections.ExtendedProperties;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
-import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@@ -57,11 +57,7 @@ public class SolrParamResourceLoader extends ResourceLoader {
@Override
public InputStream getResourceStream(String s) throws ResourceNotFoundException {
String template = templates.get(s);
- try {
- return template == null ? null : new ByteArrayInputStream(template.getBytes("UTF-8"));
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // may not happen
- }
+ return template == null ? null : new ByteArrayInputStream(template.getBytes(StandardCharsets.UTF_8));
}
@Override
diff --git a/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java b/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java
index 413d708f079..3318713045a 100644
--- a/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java
+++ b/solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java
@@ -17,21 +17,29 @@
package org.apache.solr.response;
-import org.apache.lucene.util.IOUtils;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.util.Properties;
+
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.SolrResponseBase;
-import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.velocity.Template;
import org.apache.velocity.VelocityContext;
import org.apache.velocity.app.VelocityEngine;
import org.apache.velocity.runtime.RuntimeConstants;
-import org.apache.velocity.tools.generic.*;
-
-import java.io.*;
-import java.util.Properties;
+import org.apache.velocity.tools.generic.ComparisonDateTool;
+import org.apache.velocity.tools.generic.EscapeTool;
+import org.apache.velocity.tools.generic.ListTool;
+import org.apache.velocity.tools.generic.MathTool;
+import org.apache.velocity.tools.generic.NumberTool;
+import org.apache.velocity.tools.generic.SortTool;
public class VelocityResponseWriter implements QueryResponseWriter {
@@ -132,7 +140,7 @@ public class VelocityResponseWriter implements QueryResponseWriter {
try {
is = resourceLoader.getResourceStream(propFile);
Properties props = new Properties();
- props.load(new InputStreamReader(is, IOUtils.CHARSET_UTF_8));
+ props.load(new InputStreamReader(is, StandardCharsets.UTF_8));
engine.init(props);
}
finally {
diff --git a/solr/core/build.xml b/solr/core/build.xml
index 90c4bc4a872..3d8d11f848d 100644
--- a/solr/core/build.xml
+++ b/solr/core/build.xml
@@ -32,9 +32,14 @@
diff --git a/solr/core/src/java/org/apache/solr/analytics/accumulator/BasicAccumulator.java b/solr/core/src/java/org/apache/solr/analytics/accumulator/BasicAccumulator.java
index 1e3a2db1719..304c0a2b5a0 100644
--- a/solr/core/src/java/org/apache/solr/analytics/accumulator/BasicAccumulator.java
+++ b/solr/core/src/java/org/apache/solr/analytics/accumulator/BasicAccumulator.java
@@ -18,8 +18,10 @@
package org.apache.solr.analytics.accumulator;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
+import java.util.List;
import java.util.Set;
import org.apache.lucene.index.AtomicReaderContext;
@@ -35,6 +37,8 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.schema.TrieDateField;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.SolrIndexSearcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.base.Supplier;
@@ -42,6 +46,7 @@ import com.google.common.base.Supplier;
* A BasicAccumulator manages the ValueCounters and Expressions without regard to Facets.
*/
public class BasicAccumulator extends ValueAccumulator {
+ private static final Logger log = LoggerFactory.getLogger(BasicAccumulator.class);
protected final SolrIndexSearcher searcher;
protected final AnalyticsRequest request;
protected final DocSet docs;
@@ -57,14 +62,16 @@ public class BasicAccumulator extends ValueAccumulator {
this.searcher = searcher;
this.docs = docs;
this.request = request;
- statsCollectorArraySupplier = StatsCollectorSupplierFactory.create(searcher.getSchema(), request);
+ final List exRequests = new ArrayList(request.getExpressions()); // make a copy here
+ Collections.sort(exRequests);
+ log.info("Processing request '"+request.getName()+"'");
+ statsCollectorArraySupplier = StatsCollectorSupplierFactory.create(searcher.getSchema(), exRequests);
statsCollectors = statsCollectorArraySupplier.get();
- int size = request.getExpressions().size();
+ int size = exRequests.size();
expressionNames = new String[size];
expressionStrings = new String[size];
int count = 0;
- Collections.sort(request.getExpressions());
- for (ExpressionRequest expRequest : request.getExpressions()) {
+ for (ExpressionRequest expRequest : exRequests) {
expressionNames[count] = expRequest.getName();
expressionStrings[count++] = expRequest.getExpressionString();
}
diff --git a/solr/core/src/java/org/apache/solr/analytics/accumulator/FacetingAccumulator.java b/solr/core/src/java/org/apache/solr/analytics/accumulator/FacetingAccumulator.java
index c23e63364c9..61ed6e100b9 100644
--- a/solr/core/src/java/org/apache/solr/analytics/accumulator/FacetingAccumulator.java
+++ b/solr/core/src/java/org/apache/solr/analytics/accumulator/FacetingAccumulator.java
@@ -29,6 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.TreeMap;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.Filter;
@@ -98,7 +99,7 @@ public class FacetingAccumulator extends BasicAccumulator implements FacetValueA
List rangeFreqs = request.getRangeFacets();
List queryFreqs = request.getQueryFacets();
- this.fieldFacetExpressions = new LinkedHashMap<>(fieldFreqs.size());
+ this.fieldFacetExpressions = new TreeMap<>();
this.rangeFacetExpressions = new LinkedHashMap<>(rangeFreqs.size());
this.queryFacetExpressions = new LinkedHashMap<>(queryFreqs.size());
this.fieldFacetCollectors = new LinkedHashMap<>(fieldFreqs.size());
@@ -120,8 +121,8 @@ public class FacetingAccumulator extends BasicAccumulator implements FacetValueA
final SchemaField ff = fr.getField();
final FieldFacetAccumulator facc = FieldFacetAccumulator.create(searcher, this, ff);
facetAccumulators.add(facc);
- fieldFacetExpressions.put(freq.getName(), new LinkedHashMap() );
- fieldFacetCollectors.put(freq.getName(), new LinkedHashMap());
+ fieldFacetExpressions.put(freq.getName(), new TreeMap() );
+ fieldFacetCollectors.put(freq.getName(), new TreeMap());
}
/**
* For each range and query facet request add a bucket to the corresponding
diff --git a/solr/core/src/java/org/apache/solr/analytics/expression/BaseExpression.java b/solr/core/src/java/org/apache/solr/analytics/expression/BaseExpression.java
index 3e56c89c665..1455cbcf320 100644
--- a/solr/core/src/java/org/apache/solr/analytics/expression/BaseExpression.java
+++ b/solr/core/src/java/org/apache/solr/analytics/expression/BaseExpression.java
@@ -35,7 +35,10 @@ public class BaseExpression extends Expression {
}
public Comparable getValue() {
- return statsCollector.getStat(stat);
+ if(statsCollector.getStatsList().contains(stat)) {
+ return statsCollector.getStat(stat);
+ }
+ return null;
}
}
/**
diff --git a/solr/core/src/java/org/apache/solr/analytics/request/AnalyticsStats.java b/solr/core/src/java/org/apache/solr/analytics/request/AnalyticsStats.java
index e019569fb0f..c1ec21fb15b 100644
--- a/solr/core/src/java/org/apache/solr/analytics/request/AnalyticsStats.java
+++ b/solr/core/src/java/org/apache/solr/analytics/request/AnalyticsStats.java
@@ -33,6 +33,8 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.SolrIndexSearcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Class which computes the set of {@link AnalyticsRequest}s.
@@ -43,6 +45,7 @@ public class AnalyticsStats {
protected SolrIndexSearcher searcher;
protected SolrQueryRequest req;
protected AnalyticsStatisticsCollector statsCollector;
+ private static final Logger log = LoggerFactory.getLogger(AnalyticsStats.class);
public AnalyticsStats(SolrQueryRequest req, DocSet docs, SolrParams params, AnalyticsStatisticsCollector statsCollector) {
this.req = req;
@@ -69,6 +72,10 @@ public class AnalyticsStats {
return res;
}
statsCollector.addRequests(requests.size());
+
+ // Get filter to all docs
+ Filter filter = docs.getTopFilter();
+
// Computing each Analytics Request Seperately
for( AnalyticsRequest areq : requests ){
// The Accumulator which will control the statistics generation
@@ -84,7 +91,7 @@ public class AnalyticsStats {
accumulator = FacetingAccumulator.create(searcher, docs, areq, req);
}
} catch (IOException e) {
- System.err.println(e.getMessage());
+ log.warn("Analytics request '"+areq.getName()+"' failed", e);
continue;
}
@@ -96,7 +103,6 @@ public class AnalyticsStats {
statsCollector.addQueries(((BasicAccumulator)accumulator).getNumQueries());
// Loop through the documents returned by the query and add to accumulator
- Filter filter = docs.getTopFilter();
List contexts = searcher.getTopReaderContext().leaves();
for (int leafNum = 0; leafNum < contexts.size(); leafNum++) {
AtomicReaderContext context = contexts.get(leafNum);
diff --git a/solr/core/src/java/org/apache/solr/analytics/statistics/MinMaxStatsCollector.java b/solr/core/src/java/org/apache/solr/analytics/statistics/MinMaxStatsCollector.java
index 08608861789..45cec2bc08a 100644
--- a/solr/core/src/java/org/apache/solr/analytics/statistics/MinMaxStatsCollector.java
+++ b/solr/core/src/java/org/apache/solr/analytics/statistics/MinMaxStatsCollector.java
@@ -74,7 +74,7 @@ public class MinMaxStatsCollector implements StatsCollector{
if (stat.equals("min")&&min!=null) {
return (Comparable)min.toObject();
}
- if (stat.equals("max")&&min!=null) {
+ if (stat.equals("max")&&max!=null) {
return (Comparable)max.toObject();
}
if (stat.equals("count")) {
@@ -83,7 +83,9 @@ public class MinMaxStatsCollector implements StatsCollector{
if (stat.equals("missing")) {
return new Long(missingCount);
}
+
return null;
+// throw new IllegalArgumentException("No stat named '"+stat+"' in this collector " + this);
}
public Set getStatsList() {
diff --git a/solr/core/src/java/org/apache/solr/analytics/statistics/StatsCollectorSupplierFactory.java b/solr/core/src/java/org/apache/solr/analytics/statistics/StatsCollectorSupplierFactory.java
index 7b2d14b74af..eac86643964 100644
--- a/solr/core/src/java/org/apache/solr/analytics/statistics/StatsCollectorSupplierFactory.java
+++ b/solr/core/src/java/org/apache/solr/analytics/statistics/StatsCollectorSupplierFactory.java
@@ -21,9 +21,11 @@ import java.text.ParseException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeMap;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource;
@@ -33,7 +35,6 @@ import org.apache.lucene.queries.function.valuesource.IntFieldSource;
import org.apache.lucene.queries.function.valuesource.LongFieldSource;
import org.apache.lucene.search.FieldCache;
import org.apache.solr.analytics.expression.ExpressionFactory;
-import org.apache.solr.analytics.request.AnalyticsRequest;
import org.apache.solr.analytics.request.ExpressionRequest;
import org.apache.solr.analytics.util.AnalyticsParams;
import org.apache.solr.analytics.util.AnalyticsParsers;
@@ -67,10 +68,13 @@ import org.apache.solr.schema.TrieDoubleField;
import org.apache.solr.schema.TrieFloatField;
import org.apache.solr.schema.TrieIntField;
import org.apache.solr.schema.TrieLongField;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.base.Supplier;
public class StatsCollectorSupplierFactory {
+ private static final Logger log = LoggerFactory.getLogger(StatsCollectorSupplierFactory.class);
// FunctionTypes
final static int NUMBER_TYPE = 0;
@@ -83,18 +87,18 @@ public class StatsCollectorSupplierFactory {
* Builds a Supplier that will generate identical arrays of new StatsCollectors.
*
* @param schema The Schema being used.
- * @param request The AnalyticsRequest to generate a StatsCollector[] from.
+ * @param exRequests The expression requests to generate a StatsCollector[] from.
* @return A Supplier that will return an array of new StatsCollector.
*/
@SuppressWarnings("unchecked")
- public static Supplier create(IndexSchema schema, AnalyticsRequest request) {
- final Map> collectorStats = new HashMap<>();
- final Map> collectorPercs = new HashMap<>();
- final Map collectorSources = new HashMap<>();
+ public static Supplier create(IndexSchema schema, List exRequests ) {
+ final Map> collectorStats = new TreeMap<>();
+ final Map> collectorPercs = new TreeMap<>();
+ final Map collectorSources = new TreeMap<>();
// Iterate through all expression request to make a list of ValueSource strings
// and statistics that need to be calculated on those ValueSources.
- for (ExpressionRequest expRequest : request.getExpressions()) {
+ for (ExpressionRequest expRequest : exRequests) {
String statExpression = expRequest.getExpressionString();
Set statistics = getStatistics(statExpression);
if (statistics == null) {
@@ -146,7 +150,11 @@ public class StatsCollectorSupplierFactory {
stats = new HashSet<>();
collectorStats.put(source, stats);
}
- stats.add(stat);
+ if(AnalyticsParams.STAT_PERCENTILE.equals(stat)) {
+ stats.add(stat + "_"+ arguments[0]);
+ } else {
+ stats.add(stat);
+ }
}
}
String[] keys = collectorStats.keySet().toArray(new String[0]);
@@ -168,7 +176,7 @@ public class StatsCollectorSupplierFactory {
if (percs!=null) {
collectorPercs.put(builtString, percs);
}
- for (ExpressionRequest er : request.getExpressions()) {
+ for (ExpressionRequest er : exRequests) {
er.setExpressionString(er.getExpressionString().replace(sourceStr, builtString));
}
}
@@ -182,6 +190,8 @@ public class StatsCollectorSupplierFactory {
};
}
+ log.info("Stats objects: "+collectorStats.size()+" sr="+collectorSources.size()+" pr="+collectorPercs.size() );
+
// All information is stored in final arrays so that nothing
// has to be computed when the Supplier's get() method is called.
final Set[] statsArr = collectorStats.values().toArray(new Set[0]);
diff --git a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
index 6b9e0c7eb89..af30c74410a 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
@@ -68,10 +68,11 @@ public abstract class ElectionContext {
public void cancelElection() throws InterruptedException, KeeperException {
try {
+ log.info("canceling election {}",leaderSeqPath );
zkClient.delete(leaderSeqPath, -1, true);
} catch (NoNodeException e) {
// fine
- log.warn("cancelElection did not find election node to remove");
+ log.warn("cancelElection did not find election node to remove",e);
}
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java b/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
index d1f5f96b955..88b564278fc 100644
--- a/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
+++ b/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
@@ -67,6 +67,8 @@ public class LeaderElector {
private volatile ElectionContext context;
+ private ElectionWatcher watcher;
+
public LeaderElector(SolrZkClient zkClient) {
this.zkClient = zkClient;
zkCmdExecutor = new ZkCmdExecutor(zkClient.getZkClientTimeout());
@@ -90,7 +92,7 @@ public class LeaderElector {
// get all other numbers...
final String holdElectionPath = context.electionPath + ELECTION_NODE;
List seqs = zkClient.getChildren(holdElectionPath, null, true);
-
+
sortSeqs(seqs);
List intSeqs = getSeqs(seqs);
if (intSeqs.size() == 0) {
@@ -122,31 +124,7 @@ public class LeaderElector {
return;
}
try {
- zkClient.getData(holdElectionPath + "/" + seqs.get(index),
- new Watcher() {
-
- @Override
- public void process(WatchedEvent event) {
- // session events are not change events,
- // and do not remove the watcher
- if (EventType.None.equals(event.getType())) {
- return;
- }
- // am I the next leader?
- try {
- checkIfIamLeader(seq, context, true);
- } catch (InterruptedException e) {
- // Restore the interrupted status
- Thread.currentThread().interrupt();
- log.warn("", e);
- } catch (IOException e) {
- log.warn("", e);
- } catch (Exception e) {
- log.warn("", e);
- }
- }
-
- }, null, true);
+ zkClient.getData(holdElectionPath + "/" + seqs.get(index), watcher = new ElectionWatcher(context.leaderSeqPath , seq, context) , null, true);
} catch (KeeperException.SessionExpiredException e) {
throw e;
} catch (KeeperException e) {
@@ -290,6 +268,50 @@ public class LeaderElector {
return seq;
}
+
+ private class ElectionWatcher implements Watcher {
+ final String leaderSeqPath;
+ final int seq;
+ final ElectionContext context;
+
+ private boolean canceled = false;
+
+ private ElectionWatcher(String leaderSeqPath, int seq, ElectionContext context) {
+ this.leaderSeqPath = leaderSeqPath;
+ this.seq = seq;
+ this.context = context;
+ }
+
+ void cancel(String leaderSeqPath){
+ canceled = true;
+
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+ // session events are not change events,
+ // and do not remove the watcher
+ if (EventType.None.equals(event.getType())) {
+ return;
+ }
+ if(canceled) {
+ log.info("This watcher is not active anymore {}", leaderSeqPath);
+ return;
+ }
+ try {
+ // am I the next leader?
+ checkIfIamLeader(seq, context, true);
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.warn("", e);
+ } catch (IOException e) {
+ log.warn("", e);
+ } catch (Exception e) {
+ log.warn("", e);
+ }
+ }
+ }
/**
* Set up any ZooKeeper nodes needed for leader election.
@@ -317,6 +339,8 @@ public class LeaderElector {
}
void retryElection() throws KeeperException, InterruptedException, IOException {
context.cancelElection();
+ ElectionWatcher watcher = this.watcher;
+ if(watcher!= null) watcher.cancel(context.leaderSeqPath);
joinElection(context, true);
}
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index 5e18a58fd39..94b9e0bec1a 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -70,6 +70,7 @@ public class Overseer {
public static final String ADD_ROUTING_RULE = "addroutingrule";
public static final String REMOVE_ROUTING_RULE = "removeroutingrule";
public static final String STATE = "state";
+ public static final String QUIT = "quit";
public static final int STATE_UPDATE_DELAY = 1500; // delay between cloud state updates
public static final String CREATESHARD = "createshard";
@@ -200,85 +201,132 @@ public class Overseer {
}
log.info("Starting to work on the main queue");
- while (!this.isClosed) {
- isLeader = amILeader();
- if (LeaderStatus.NO == isLeader) {
- break;
- }
- else if (LeaderStatus.YES != isLeader) {
- log.debug("am_i_leader unclear {}", isLeader);
- continue; // not a no, not a yes, try ask again
- }
- DistributedQueue.QueueEvent head = null;
- try {
- head = stateUpdateQueue.peek(true);
- } catch (KeeperException e) {
- if (e.code() == KeeperException.Code.SESSIONEXPIRED) {
- log.warn(
- "Solr cannot talk to ZK, exiting Overseer main queue loop", e);
- return;
+ try {
+ while (!this.isClosed) {
+ isLeader = amILeader();
+ if (LeaderStatus.NO == isLeader) {
+ break;
}
- log.error("Exception in Overseer main queue loop", e);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- return;
-
- } catch (Exception e) {
- log.error("Exception in Overseer main queue loop", e);
- }
- synchronized (reader.getUpdateLock()) {
+ else if (LeaderStatus.YES != isLeader) {
+ log.debug("am_i_leader unclear {}", isLeader);
+ continue; // not a no, not a yes, try ask again
+ }
+ DistributedQueue.QueueEvent head = null;
try {
- reader.updateClusterState(true);
- ClusterState clusterState = reader.getClusterState();
-
- while (head != null) {
- final ZkNodeProps message = ZkNodeProps.load(head.getBytes());
- final String operation = message.getStr(QUEUE_OPERATION);
- final TimerContext timerContext = stats.time(operation);
- try {
- clusterState = processMessage(clusterState, message, operation);
- stats.success(operation);
- } catch (Exception e) {
- // generally there is nothing we can do - in most cases, we have
- // an issue that will fail again on retry or we cannot communicate with
- // ZooKeeper in which case another Overseer should take over
- // TODO: if ordering for the message is not important, we could
- // track retries and put it back on the end of the queue
- log.error("Overseer could not process the current clusterstate state update message, skipping the message.", e);
- stats.error(operation);
- } finally {
- timerContext.stop();
- }
- workQueue.offer(head.getBytes());
-
- stateUpdateQueue.poll();
-
- if (System.nanoTime() - lastUpdatedTime > TimeUnit.NANOSECONDS.convert(STATE_UPDATE_DELAY, TimeUnit.MILLISECONDS)) break;
-
- // if an event comes in the next 100ms batch it together
- head = stateUpdateQueue.peek(100);
- }
- lastUpdatedTime = System.nanoTime();
- zkClient.setData(ZkStateReader.CLUSTER_STATE,
- ZkStateReader.toJSON(clusterState), true);
- // clean work queue
- while (workQueue.poll() != null) ;
-
+ head = stateUpdateQueue.peek(true);
} catch (KeeperException e) {
if (e.code() == KeeperException.Code.SESSIONEXPIRED) {
- log.warn("Solr cannot talk to ZK, exiting Overseer main queue loop", e);
+ log.warn(
+ "Solr cannot talk to ZK, exiting Overseer main queue loop", e);
return;
}
log.error("Exception in Overseer main queue loop", e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return;
-
+
} catch (Exception e) {
log.error("Exception in Overseer main queue loop", e);
}
+ synchronized (reader.getUpdateLock()) {
+ try {
+ reader.updateClusterState(true);
+ ClusterState clusterState = reader.getClusterState();
+
+ while (head != null) {
+ final ZkNodeProps message = ZkNodeProps.load(head.getBytes());
+ final String operation = message.getStr(QUEUE_OPERATION);
+ final TimerContext timerContext = stats.time(operation);
+ try {
+ clusterState = processMessage(clusterState, message, operation);
+ stats.success(operation);
+ } catch (Exception e) {
+ // generally there is nothing we can do - in most cases, we have
+ // an issue that will fail again on retry or we cannot communicate with
+ // ZooKeeper in which case another Overseer should take over
+ // TODO: if ordering for the message is not important, we could
+ // track retries and put it back on the end of the queue
+ log.error("Overseer could not process the current clusterstate state update message, skipping the message.", e);
+ stats.error(operation);
+ } finally {
+ timerContext.stop();
+ }
+ workQueue.offer(head.getBytes());
+
+ stateUpdateQueue.poll();
+
+ if (isClosed || System.nanoTime() - lastUpdatedTime > TimeUnit.NANOSECONDS.convert(STATE_UPDATE_DELAY, TimeUnit.MILLISECONDS)) break;
+
+ // if an event comes in the next 100ms batch it together
+ head = stateUpdateQueue.peek(100);
+ }
+ lastUpdatedTime = System.nanoTime();
+ zkClient.setData(ZkStateReader.CLUSTER_STATE,
+ ZkStateReader.toJSON(clusterState), true);
+ // clean work queue
+ while (workQueue.poll() != null) ;
+
+ } catch (KeeperException e) {
+ if (e.code() == KeeperException.Code.SESSIONEXPIRED) {
+ log.warn("Solr cannot talk to ZK, exiting Overseer main queue loop", e);
+ return;
+ }
+ log.error("Exception in Overseer main queue loop", e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return;
+
+ } catch (Exception e) {
+ log.error("Exception in Overseer main queue loop", e);
+ }
+ }
+
}
-
+ } finally {
+ log.info("Overseer Loop exiting : {}", LeaderElector.getNodeName(myId));
+ new Thread("OverseerExitThread"){
+ //do this in a separate thread because any wait is interrupted in this main thread
+ @Override
+ public void run() {
+ checkIfIamStillLeader();
+ }
+ }.start();
+ }
+ }
+
+ private void checkIfIamStillLeader() {
+ org.apache.zookeeper.data.Stat stat = new org.apache.zookeeper.data.Stat();
+ String path = "/overseer_elect/leader";
+ byte[] data = null;
+ try {
+ data = zkClient.getData(path, null, stat, true);
+ } catch (Exception e) {
+ log.error("could not read the data" ,e);
+ return;
+ }
+ Map m = (Map) ZkStateReader.fromJSON(data);
+ String id = (String) m.get("id");
+ if(overseerCollectionProcessor.getId().equals(id)){
+ try {
+ log.info("I'm exiting , but I'm still the leader");
+ zkClient.delete(path,stat.getVersion(),true);
+ } catch (KeeperException.BadVersionException e) {
+ //no problem ignore it some other Overseer has already taken over
+ } catch (Exception e) {
+ log.error("Could not delete my leader node ", e);
+ } finally {
+ try {
+ if(zkController !=null && !zkController.getCoreContainer().isShutDown()){
+ zkController.rejoinOverseerElection();
+ }
+
+ } catch (Exception e) {
+ log.error("error canceling overseer election election ",e);
+ }
+ }
+
+ } else{
+ log.info("somebody else has already taken up the overseer position");
}
}
@@ -324,7 +372,11 @@ public class Overseer {
clusterState = removeRoutingRule(clusterState, message);
} else if(CLUSTERPROP.isEqual(operation)){
handleProp(message);
- } else {
+ } else if( QUIT.equals(operation)){
+ log.info("Quit command received {}", LeaderElector.getNodeName(myId));
+ overseerCollectionProcessor.close();
+ close();
+ } else{
throw new RuntimeException("unknown operation:" + operation
+ " contents:" + message.getProperties());
}
@@ -1107,15 +1159,18 @@ public class Overseer {
private String adminPath;
- private OverseerCollectionProcessor ocp;
+ private OverseerCollectionProcessor overseerCollectionProcessor;
+
+ private ZkController zkController;
private Stats stats;
// overseer not responsible for closing reader
- public Overseer(ShardHandler shardHandler, String adminPath, final ZkStateReader reader) throws KeeperException, InterruptedException {
+ public Overseer(ShardHandler shardHandler, String adminPath, final ZkStateReader reader, ZkController zkController) throws KeeperException, InterruptedException {
this.reader = reader;
this.shardHandler = shardHandler;
this.adminPath = adminPath;
+ this.zkController = zkController;
this.stats = new Stats();
}
@@ -1130,8 +1185,8 @@ public class Overseer {
ThreadGroup ccTg = new ThreadGroup("Overseer collection creation process.");
- ocp = new OverseerCollectionProcessor(reader, id, shardHandler, adminPath, stats);
- ccThread = new OverseerThread(ccTg, ocp, "Overseer-" + id);
+ overseerCollectionProcessor = new OverseerCollectionProcessor(reader, id, shardHandler, adminPath, stats);
+ ccThread = new OverseerThread(ccTg, overseerCollectionProcessor, "Overseer-" + id);
ccThread.setDaemon(true);
updaterThread.start();
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
index a07acdb7816..f4abda8d5b9 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
@@ -88,6 +88,8 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.CLUSTERSTATUS;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.LIST;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.OVERSEERSTATUS;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.REMOVEROLE;
@@ -136,7 +138,6 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread {
public static final String COLL_PROP_PREFIX = "property.";
-
public static final Set KNOWN_CLUSTER_PROPS = ImmutableSet.of(ZkStateReader.LEGACY_CLOUD, ZkStateReader.URL_SCHEME);
public static final Map COLL_PROPS = ZkNodeProps.makeMap(
@@ -220,6 +221,7 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread {
}
QueueEvent head = workQueue.peek(true);
+ if(isClosed) break;
final ZkNodeProps message = ZkNodeProps.load(head.getBytes());
final String asyncId = (message.containsKey(ASYNC) && message.get(ASYNC) != null) ? (String) message.get(ASYNC) : null;
@@ -287,7 +289,7 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread {
}
private void prioritizeOverseerNodes() throws KeeperException, InterruptedException {
- log.info("prioritizing overseer nodes");
+ log.info("prioritizing overseer nodes at {}", LeaderElector.getNodeName(myId));
SolrZkClient zk = zkStateReader.getZkClient();
if(!zk.exists(ZkStateReader.ROLES,true))return;
Map m = (Map) ZkStateReader.fromJSON(zk.getData(ZkStateReader.ROLES, null, new Stat(), true));
@@ -299,6 +301,7 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread {
List nodeNames = getSortedOverseerNodeNames(zk);
if(nodeNames.size()<2) return;
+ boolean designateIsInFront = overseerDesignates.contains( nodeNames.get(0));
//
ArrayList nodesTobePushedBack = new ArrayList<>();
@@ -306,25 +309,25 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread {
List availableDesignates = new ArrayList<>();
log.info("sorted nodes {}", nodeNames);//TODO to be removed
- for (int i = 0; i < nodeNames.size(); i++) {
+ for (int i = 1; i < nodeNames.size(); i++) {
String s = nodeNames.get(i);
if (overseerDesignates.contains(s)) {
availableDesignates.add(s);
- for(int j=0;j1) break;
+ if(availableDesignates.size()>1) break;//we don't need to line up more than 2 designates
}
if(!availableDesignates.isEmpty()){
- for (int i = nodesTobePushedBack.size() - 1; i >= 0; i--) {
- String s = nodesTobePushedBack.get(i);
+ for (String s : nodesTobePushedBack) {
log.info("pushing back {} ", s);
invokeOverseerOp(s, "rejoin");
}
@@ -358,18 +361,22 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread {
log.warn("available designates and current state {} {} ", availableDesignates, getSortedOverseerNodeNames(zk));
}
- } else {
+ } else if(!designateIsInFront) {
log.warn("No overseer designates are available, overseerDesignates: {}, live nodes : {}",overseerDesignates,nodeNames);
return;
}
String leaderNode = getLeaderNode(zkStateReader.getZkClient());
if(leaderNode ==null) return;
- if(!overseerDesignates.contains(leaderNode) && !availableDesignates.isEmpty()){
- //this means there are designated Overseer nodes and I am not one of them , kill myself
- String newLeader = availableDesignates.get(0);
- log.info("I am not an overseerdesignate , forcing a new leader {} ", newLeader);
- invokeOverseerOp(newLeader, "leader");
+ if(!overseerDesignates.contains(leaderNode) ){
+ List sortedNodes = getSortedOverseerNodeNames(zk);
+
+ if(leaderNode.equals(sortedNodes.get(0)) || // I am leader and I am in front of the queue
+ overseerDesignates.contains(sortedNodes.get(0))) {// I am leader but somebody else is in the front , Screwed up leader election
+ //this means there are I am not a designate and the next guy is lined up to become the leader, kill myself
+ log.info("I am not an overseer designate , forcing myself out {} ", leaderNode);
+ Overseer.getInQueue(zkStateReader.getZkClient()).offer(ZkStateReader.toJSON(new ZkNodeProps(Overseer.QUEUE_OPERATION, Overseer.QUIT)));
+ }
}
}
@@ -470,13 +477,13 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread {
processRoleCommand(message, operation);
} else if (ADDREPLICA.isEqual(operation)) {
addReplica(zkStateReader.getClusterState(), message, results);
- } else if (REQUESTSTATUS.equals(operation)) {
- requestStatus(message, results);
} else if (OVERSEERSTATUS.isEqual(operation)) {
getOverseerStatus(message, results);
- }
-
- else {
+ } else if(LIST.isEqual(operation)) {
+ listCollections(zkStateReader.getClusterState(), results);
+ } else if (CLUSTERSTATUS.isEqual(operation)) {
+ getClusterStatus(zkStateReader.getClusterState(), message, results);
+ } else {
throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:"
+ operation);
}
@@ -567,6 +574,131 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread {
}
+ private void getClusterStatus(ClusterState clusterState, ZkNodeProps message, NamedList results) {
+ String collection = message.getStr(ZkStateReader.COLLECTION_PROP);
+
+ // read aliases
+ Aliases aliases = zkStateReader.getAliases();
+ Map> collectionVsAliases = new HashMap<>();
+ Map aliasVsCollections = aliases.getCollectionAliasMap();
+ if (aliasVsCollections != null) {
+ for (Map.Entry entry : aliasVsCollections.entrySet()) {
+ List colls = StrUtils.splitSmart(entry.getValue(), ',');
+ String alias = entry.getKey();
+ for (String coll : colls) {
+ if (collection == null || collection.equals(coll)) {
+ List list = collectionVsAliases.get(coll);
+ if (list == null) {
+ list = new ArrayList<>();
+ collectionVsAliases.put(coll, list);
+ }
+ list.add(alias);
+ }
+ }
+ }
+ }
+
+ // convert cluster state into a map of writable types
+ byte[] bytes = ZkStateReader.toJSON(clusterState);
+ Map stateMap = (Map) ZkStateReader.fromJSON(bytes);
+
+ String shard = message.getStr(ZkStateReader.SHARD_ID_PROP);
+ NamedList collectionProps = new SimpleOrderedMap();
+ if (collection == null) {
+ Set collections = clusterState.getCollections();
+ for (String name : collections) {
+ Map collectionStatus = getCollectionStatus(stateMap, name, shard);
+ if (collectionVsAliases.containsKey(name) && !collectionVsAliases.get(name).isEmpty()) {
+ collectionStatus.put("aliases", collectionVsAliases.get(name));
+ }
+ collectionProps.add(name, collectionStatus);
+ }
+ } else {
+ String routeKey = message.getStr(ShardParams._ROUTE_);
+ if (routeKey == null) {
+ Map collectionStatus = getCollectionStatus(stateMap, collection, shard);
+ if (collectionVsAliases.containsKey(collection) && !collectionVsAliases.get(collection).isEmpty()) {
+ collectionStatus.put("aliases", collectionVsAliases.get(collection));
+ }
+ collectionProps.add(collection, collectionStatus);
+ } else {
+ DocCollection docCollection = clusterState.getCollection(collection);
+ DocRouter router = docCollection.getRouter();
+ Collection slices = router.getSearchSlices(routeKey, null, docCollection);
+ String s = "";
+ for (Slice slice : slices) {
+ s += slice.getName() + ",";
+ }
+ if (shard != null) {
+ s += shard;
+ }
+ Map collectionStatus = getCollectionStatus(stateMap, collection, s);
+ if (collectionVsAliases.containsKey(collection) && !collectionVsAliases.get(collection).isEmpty()) {
+ collectionStatus.put("aliases", collectionVsAliases.get(collection));
+ }
+ collectionProps.add(collection, collectionStatus);
+ }
+ }
+
+
+ NamedList clusterStatus = new SimpleOrderedMap<>();
+ clusterStatus.add("collections", collectionProps);
+
+ // read cluster properties
+ Map clusterProps = zkStateReader.getClusterProps();
+ if (clusterProps != null && !clusterProps.isEmpty()) {
+ clusterStatus.add("properties", clusterProps);
+ }
+
+ // add the alias map too
+ if (aliasVsCollections != null && !aliasVsCollections.isEmpty()) {
+ clusterStatus.add("aliases", aliasVsCollections);
+ }
+
+ results.add("cluster", clusterStatus);
+ }
+
+ /**
+ * Get collection status from cluster state.
+ * Can return collection status by given shard name.
+ *
+ *
+ * @param clusterState cloud state map parsed from JSON-serialized {@link ClusterState}
+ * @param name collection name
+ * @param shardStr comma separated shard names
+ * @return map of collection properties
+ */
+ private Map getCollectionStatus(Map clusterState, String name, String shardStr) {
+ Map docCollection = (Map) clusterState.get(name);
+ if (docCollection == null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Collection: " + name + " not found");
+ }
+ if (shardStr == null) {
+ return docCollection;
+ } else {
+ Map shards = (Map) docCollection.get("shards");
+ Map selected = new HashMap<>();
+ List selectedShards = Arrays.asList(shardStr.split(","));
+ for (String selectedShard : selectedShards) {
+ if (!shards.containsKey(selectedShard)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Collection: " + name + " shard: " + selectedShard + " not found");
+ }
+ selected.put(selectedShard, shards.get(selectedShard));
+ docCollection.put("shards", selected);
+ }
+ return docCollection;
+ }
+ }
+
+ private void listCollections(ClusterState clusterState, NamedList results) {
+ Set collections = clusterState.getCollections();
+ List collectionList = new ArrayList();
+ for (String collection : collections) {
+ collectionList.add(collection);
+ }
+ results.add("collections", collectionList);
+ }
+
private void processRoleCommand(ZkNodeProps message, String operation) throws KeeperException, InterruptedException {
SolrZkClient zkClient = zkStateReader.getZkClient();
Map roles = null;
@@ -1372,40 +1504,6 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread {
} while (srsp != null);
}
- private void requestStatus(ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException {
- log.info("Request status invoked");
- String requestId = message.getStr(REQUESTID);
-
- // Special taskId (-1), clears up the request state maps.
- if(requestId.equals("-1")) {
- completedMap.clear();
- failureMap.clear();
- return;
- }
-
- if(completedMap.contains(requestId)) {
- SimpleOrderedMap success = new SimpleOrderedMap();
- success.add("state", "completed");
- success.add("msg", "found " + requestId + " in completed tasks");
- results.add("status", success);
- } else if (runningMap.contains(requestId)) {
- SimpleOrderedMap success = new SimpleOrderedMap();
- success.add("state", "running");
- success.add("msg", "found " + requestId + " in submitted tasks");
- results.add("status", success);
- } else if (failureMap.contains(requestId)) {
- SimpleOrderedMap success = new SimpleOrderedMap();
- success.add("state", "failed");
- success.add("msg", "found " + requestId + " in failed tasks");
- results.add("status", success);
- } else {
- SimpleOrderedMap failure = new SimpleOrderedMap();
- failure.add("state", "notfound");
- failure.add("msg", "Did not find taskid [" + requestId + "] in any tasks queue");
- results.add("status", failure);
- }
- }
-
private void deleteShard(ClusterState clusterState, ZkNodeProps message, NamedList results) {
log.info("Delete shard invoked");
String collection = message.getStr(ZkStateReader.COLLECTION_PROP);
@@ -2277,5 +2375,9 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread {
} while (srsp != null);
} while(true);
}
+ String getId(){
+ return myId;
+ }
+
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java b/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java
index dc4e76132d2..8a00887bce7 100644
--- a/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java
@@ -25,11 +25,11 @@ import java.io.InputStreamReader;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
+import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.Properties;
import java.util.Map.Entry;
-import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.SolrException;
import org.apache.zookeeper.server.ServerConfig;
import org.apache.zookeeper.server.ZooKeeperServerMain;
@@ -179,7 +179,7 @@ class SolrZkServerProps extends QuorumPeerConfig {
Properties cfg = new Properties();
FileInputStream in = new FileInputStream(configFile);
try {
- cfg.load(new InputStreamReader(in, IOUtils.CHARSET_UTF_8));
+ cfg.load(new InputStreamReader(in, StandardCharsets.UTF_8));
} finally {
in.close();
}
@@ -461,7 +461,7 @@ class SolrZkServerProps extends QuorumPeerConfig {
+ " file is missing");
}
- BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(myIdFile), IOUtils.CHARSET_UTF_8));
+ BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(myIdFile), StandardCharsets.UTF_8));
String myIdString;
try {
myIdString = br.readLine();
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
index 4190aa3f9a4..ef5c469e894 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
@@ -20,10 +20,12 @@ import org.apache.zookeeper.data.ACL;
import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
+
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.TimeoutException;
@@ -253,7 +255,7 @@ public class ZkCLI {
System.out.println("-" + PUT + " requires two args - the path to create and the data string");
System.exit(1);
}
- zkClient.create(arglist.get(0).toString(), arglist.get(1).toString().getBytes("UTF-8"),
+ zkClient.create(arglist.get(0).toString(), arglist.get(1).toString().getBytes(StandardCharsets.UTF_8),
acl, CreateMode.PERSISTENT, true);
} else if (line.getOptionValue(CMD).equals(PUT_FILE)) {
List arglist = line.getArgList();
@@ -276,7 +278,7 @@ public class ZkCLI {
System.exit(1);
}
byte [] data = zkClient.getData(arglist.get(0).toString(), null, null, true);
- System.out.println(new String(data, "UTF-8"));
+ System.out.println(new String(data, StandardCharsets.UTF_8));
} else if (line.getOptionValue(CMD).equals(GET_FILE)) {
List arglist = line.getArgList();
if (arglist.size() != 2) {
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index 1bd3acfd7bd..aa57e689337 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -304,18 +304,6 @@ public final class ZkController {
return leaderConflictResolveWait;
}
- public void forceOverSeer(){
- try {
- zkClient.delete("/overseer_elect/leader",-1, true);
- log.info("Forcing me to be leader {} ", getBaseUrl());
- overseerElector.getContext().runLeaderProcess(true, Overseer.STATE_UPDATE_DELAY + 100);
- } catch (Exception e) {
- throw new SolrException(ErrorCode.SERVER_ERROR, " Error becoming overseer ",e);
-
- }
-
- }
-
private void registerAllCoresAsDown(
final CurrentCoreDescriptorProvider registerOnReconnect, boolean updateLastPublished) {
List descriptors = registerOnReconnect
@@ -558,7 +546,7 @@ public final class ZkController {
adminPath = cc.getAdminPath();
overseerElector = new LeaderElector(zkClient);
- this.overseer = new Overseer(shardHandler, adminPath, zkStateReader);
+ this.overseer = new Overseer(shardHandler, adminPath, zkStateReader,this);
ElectionContext context = new OverseerElectionContext(zkClient, overseer, getNodeName());
overseerElector.setup(context);
overseerElector.joinElection(context, false);
@@ -1626,7 +1614,7 @@ public final class ZkController {
return hostName + ':' + hostPort + '_' +
URLEncoder.encode(trimLeadingAndTrailingSlashes(hostContext), "UTF-8");
} catch (UnsupportedEncodingException e) {
- throw new IllegalStateException("JVM Does not seem to support UTF-8", e);
+ throw new Error("JVM Does not seem to support UTF-8", e);
}
}
@@ -1679,4 +1667,8 @@ public final class ZkController {
}
}
+ CoreContainer getCoreContainer(){
+ return cc;
+ }
+
}
diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSolr.java b/solr/core/src/java/org/apache/solr/core/ConfigSolr.java
index 0356698596e..dae776ddc1f 100644
--- a/solr/core/src/java/org/apache/solr/core/ConfigSolr.java
+++ b/solr/core/src/java/org/apache/solr/core/ConfigSolr.java
@@ -17,7 +17,6 @@ package org.apache.solr.core;
* limitations under the License.
*/
-import com.google.common.base.Charsets;
import com.google.common.io.ByteStreams;
import org.apache.commons.io.IOUtils;
@@ -42,6 +41,7 @@ import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
@@ -77,19 +77,17 @@ public abstract class ConfigSolr {
}
public static ConfigSolr fromString(SolrResourceLoader loader, String xml) {
- return fromInputStream(loader, new ByteArrayInputStream(xml.getBytes(Charsets.UTF_8)));
+ return fromInputStream(loader, new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)));
}
public static ConfigSolr fromInputStream(SolrResourceLoader loader, InputStream is) {
try {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- ByteStreams.copy(is, baos);
- String originalXml = IOUtils.toString(new ByteArrayInputStream(baos.toByteArray()), "UTF-8");
- ByteArrayInputStream dup = new ByteArrayInputStream(baos.toByteArray());
+ byte[] buf = IOUtils.toByteArray(is);
+ String originalXml = new String(buf, StandardCharsets.UTF_8);
+ ByteArrayInputStream dup = new ByteArrayInputStream(buf);
Config config = new Config(loader, null, new InputSource(dup), null, false);
return fromConfig(config, originalXml);
- }
- catch (Exception e) {
+ } catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
@@ -241,7 +239,7 @@ public abstract class ConfigSolr {
}
public ConfigSetService createCoreConfigService(SolrResourceLoader loader, ZkController zkController) {
- if (getZkHost() != null)
+ if (getZkHost() != null || System.getProperty("zkRun") != null)
return new CloudConfigSetService(loader, zkController);
if (hasSchemaCache())
return new ConfigSetService.SchemaCaching(loader, getConfigSetBaseDirectory());
diff --git a/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java b/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java
index b9776fdfa17..df5f9b1227d 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java
@@ -31,6 +31,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.util.Locale;
import java.util.Properties;
@@ -213,7 +214,7 @@ public class CoreDescriptor {
try {
in = new FileInputStream(propertiesFile);
Properties externalProps = new Properties();
- externalProps.load(new InputStreamReader(in, "UTF-8"));
+ externalProps.load(new InputStreamReader(in, StandardCharsets.UTF_8));
coreProperties.putAll(externalProps);
} catch (IOException e) {
String message = String.format(Locale.ROOT, "Could not load properties from %s: %s:",
diff --git a/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java b/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
index 104effbe6fb..e2a59429ac6 100644
--- a/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
+++ b/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
@@ -17,7 +17,6 @@ package org.apache.solr.core;
* limitations under the License.
*/
-import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import org.apache.solr.common.SolrException;
import org.apache.solr.util.IOUtils;
@@ -31,6 +30,7 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Properties;
@@ -79,7 +79,7 @@ public class CorePropertiesLocator implements CoresLocator {
Writer os = null;
try {
propfile.getParentFile().mkdirs();
- os = new OutputStreamWriter(new FileOutputStream(propfile), Charsets.UTF_8);
+ os = new OutputStreamWriter(new FileOutputStream(propfile), StandardCharsets.UTF_8);
p.store(os, "Written by CorePropertiesLocator");
}
catch (IOException e) {
@@ -147,7 +147,7 @@ public class CorePropertiesLocator implements CoresLocator {
File instanceDir = propertiesFile.getParentFile();
Properties coreProperties = new Properties();
fis = new FileInputStream(propertiesFile);
- coreProperties.load(new InputStreamReader(fis, Charsets.UTF_8));
+ coreProperties.load(new InputStreamReader(fis, StandardCharsets.UTF_8));
String name = createName(coreProperties, instanceDir);
return new CoreDescriptor(cc, name, instanceDir.getAbsolutePath(), coreProperties);
}
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 3109fd1be6c..32a021aa755 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -99,6 +99,7 @@ import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
+
import java.io.Closeable;
import java.io.File;
import java.io.FileNotFoundException;
@@ -108,6 +109,7 @@ import java.io.InputStreamReader;
import java.io.Writer;
import java.lang.reflect.Constructor;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -274,7 +276,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
if (input != null) {
final InputStream is = new PropertiesInputStream(input);
try {
- p.load(new InputStreamReader(is, "UTF-8"));
+ p.load(new InputStreamReader(is, StandardCharsets.UTF_8));
String s = p.getProperty("index");
if (s != null && s.trim().length() > 0) {
diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
index bb375a9acbc..bc95767e5c8 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
@@ -48,6 +48,7 @@ import javax.naming.Context;
import javax.naming.InitialContext;
import javax.naming.NamingException;
import javax.naming.NoInitialContextException;
+
import java.io.Closeable;
import java.io.File;
import java.io.FileFilter;
@@ -61,6 +62,7 @@ import java.net.URL;
import java.net.URLClassLoader;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -93,7 +95,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
private final List waitingForCore = Collections.synchronizedList(new ArrayList());
private final List infoMBeans = Collections.synchronizedList(new ArrayList());
private final List waitingForResources = Collections.synchronizedList(new ArrayList());
- private static final Charset UTF_8 = Charset.forName("UTF-8");
+ private static final Charset UTF_8 = StandardCharsets.UTF_8;
//TODO: Solr5. Remove this completely when you obsolete putting tags in solr.xml (See Solr-4196)
private final Properties coreProperties;
diff --git a/solr/core/src/java/org/apache/solr/core/SolrXMLCoresLocator.java b/solr/core/src/java/org/apache/solr/core/SolrXMLCoresLocator.java
index bd59ad80f4d..bd7052341f4 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrXMLCoresLocator.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrXMLCoresLocator.java
@@ -17,7 +17,6 @@ package org.apache.solr.core;
* limitations under the License.
*/
-import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableList;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
@@ -29,6 +28,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -157,7 +157,7 @@ public class SolrXMLCoresLocator implements CoresLocator {
FileOutputStream fos = null;
try {
fos = new FileOutputStream(file);
- writer = new OutputStreamWriter(fos, Charsets.UTF_8);
+ writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8);
writer.write(xml);
writer.close();
logger.info("Persisted core descriptions to {}", file.getAbsolutePath());
diff --git a/solr/core/src/java/org/apache/solr/core/SolrXMLSerializer.java b/solr/core/src/java/org/apache/solr/core/SolrXMLSerializer.java
index c1cd720b80c..6b751a4db8a 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrXMLSerializer.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrXMLSerializer.java
@@ -28,6 +28,7 @@ import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
+
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
@@ -37,6 +38,7 @@ import java.io.OutputStreamWriter;
import java.io.StringWriter;
import java.io.Writer;
import java.nio.channels.FileChannel;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -169,7 +171,7 @@ public class SolrXMLSerializer {
tmpFile = File.createTempFile("solr", ".xml", file.getParentFile());
java.io.FileOutputStream out = new java.io.FileOutputStream(tmpFile);
- Writer writer = new BufferedWriter(new OutputStreamWriter(out, "UTF-8"));
+ Writer writer = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8));
try {
persist(writer, solrXMLDef);
} finally {
diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
index 5da16a4a8ce..5298a4c9c5b 100644
--- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
@@ -26,6 +26,7 @@ import java.io.OutputStream;
import java.io.Writer;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
+import java.nio.charset.StandardCharsets;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -81,8 +82,6 @@ import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.lucene.util.IOUtils.CHARSET_UTF_8;
-
/**
* A Handler which provides a REST API for replication and serves replication requests from Slaves.
* When running on the master, it provides the following commands
Get the current replicable index version
@@ -823,7 +822,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
try {
final InputStream is = new PropertiesInputStream(input);
Properties props = new Properties();
- props.load(new InputStreamReader(is, CHARSET_UTF_8));
+ props.load(new InputStreamReader(is, StandardCharsets.UTF_8));
return props;
} finally {
input.close();
diff --git a/solr/core/src/java/org/apache/solr/handler/SnapPuller.java b/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
index 3b2caf07771..65da7932737 100644
--- a/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
+++ b/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
@@ -26,6 +26,7 @@ import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
+import java.nio.charset.StandardCharsets;
import java.nio.file.NoSuchFileException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@@ -90,7 +91,6 @@ import org.eclipse.jetty.util.log.Log;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.lucene.util.IOUtils.CHARSET_UTF_8;
import static org.apache.solr.handler.ReplicationHandler.ALIAS;
import static org.apache.solr.handler.ReplicationHandler.CHECKSUM;
import static org.apache.solr.handler.ReplicationHandler.CMD_DETAILS;
@@ -604,7 +604,7 @@ public class SnapPuller {
}
final IndexOutput out = dir.createOutput(REPLICATION_PROPERTIES, DirectoryFactory.IOCONTEXT_NO_CACHE);
- Writer outFile = new OutputStreamWriter(new PropertiesOutputStream(out), CHARSET_UTF_8);
+ Writer outFile = new OutputStreamWriter(new PropertiesOutputStream(out), StandardCharsets.UTF_8);
try {
props.store(outFile, "Replication details");
dir.sync(Collections.singleton(REPLICATION_PROPERTIES));
@@ -945,7 +945,7 @@ public class SnapPuller {
final InputStream is = new PropertiesInputStream(input);
try {
- p.load(new InputStreamReader(is, CHARSET_UTF_8));
+ p.load(new InputStreamReader(is, StandardCharsets.UTF_8));
} catch (Exception e) {
LOG.error("Unable to load " + SnapPuller.INDEX_PROPERTIES, e);
} finally {
@@ -961,7 +961,7 @@ public class SnapPuller {
p.put("index", tmpIdxDirName);
Writer os = null;
try {
- os = new OutputStreamWriter(new PropertiesOutputStream(out), CHARSET_UTF_8);
+ os = new OutputStreamWriter(new PropertiesOutputStream(out), StandardCharsets.UTF_8);
p.store(os, SnapPuller.INDEX_PROPERTIES);
dir.sync(Collections.singleton(INDEX_PROPERTIES));
} catch (Exception e) {
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index d1d2f7b2d0a..46acf578892 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -211,6 +211,14 @@ public class CollectionsHandler extends RequestHandlerBase {
this.handleOverseerStatus(req, rsp);
break;
}
+ case LIST: {
+ this.handleListAction(req, rsp);
+ break;
+ }
+ case CLUSTERSTATUS: {
+ this.handleClusterStatus(req, rsp);
+ break;
+ }
default: {
throw new RuntimeException("Unknown action: " + action);
}
@@ -257,11 +265,43 @@ public class CollectionsHandler extends RequestHandlerBase {
private void handleRequestStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
log.debug("REQUESTSTATUS action invoked: " + req.getParamString());
req.getParams().required().check(REQUESTID);
- Map props = new HashMap();
- props.put(Overseer.QUEUE_OPERATION, OverseerCollectionProcessor.REQUESTSTATUS);
- props.put(REQUESTID, req.getParams().get(REQUESTID));
- ZkNodeProps m = new ZkNodeProps(props);
- handleResponse(OverseerCollectionProcessor.REQUESTSTATUS, m, rsp);
+
+ String requestId = req.getParams().get(REQUESTID);
+
+ if (requestId.equals("-1")) {
+ // Special taskId (-1), clears up the request state maps.
+ if(requestId.equals("-1")) {
+ coreContainer.getZkController().getOverseerCompletedMap().clear();
+ coreContainer.getZkController().getOverseerFailureMap().clear();
+ return;
+ }
+ } else {
+ NamedList results = new NamedList<>();
+ if (coreContainer.getZkController().getOverseerCompletedMap().contains(requestId)) {
+ SimpleOrderedMap success = new SimpleOrderedMap();
+ success.add("state", "completed");
+ success.add("msg", "found " + requestId + " in completed tasks");
+ results.add("status", success);
+ } else if (coreContainer.getZkController().getOverseerRunningMap().contains(requestId)) {
+ SimpleOrderedMap success = new SimpleOrderedMap();
+ success.add("state", "running");
+ success.add("msg", "found " + requestId + " in submitted tasks");
+ results.add("status", success);
+ } else if (coreContainer.getZkController().getOverseerFailureMap().contains(requestId)) {
+ SimpleOrderedMap success = new SimpleOrderedMap();
+ success.add("state", "failed");
+ success.add("msg", "found " + requestId + " in failed tasks");
+ results.add("status", success);
+ } else {
+ SimpleOrderedMap failure = new SimpleOrderedMap();
+ failure.add("state", "notfound");
+ failure.add("msg", "Did not find taskid [" + requestId + "] in any tasks queue");
+ results.add("status", failure);
+ }
+ SolrResponse response = new OverseerSolrResponse(results);
+
+ rsp.getValues().addAll(response.getResponse());
+ }
}
private void handleResponse(String operation, ZkNodeProps m,
@@ -574,6 +614,36 @@ public class CollectionsHandler extends RequestHandlerBase {
handleResponse(CollectionAction.ADDREPLICA.toString(), m, rsp);
}
+ /**
+ * Handle cluster status request.
+ * Can return status per specific collection/shard or per all collections.
+ *
+ * @param req solr request
+ * @param rsp solr response
+ */
+ private void handleClusterStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
+ Map props = new HashMap<>();
+ props.put(Overseer.QUEUE_OPERATION, CollectionAction.CLUSTERSTATUS.toLower());
+ copyIfNotNull(req.getParams(), props, COLLECTION_PROP, SHARD_ID_PROP, ShardParams._ROUTE_);
+ handleResponse(CollectionAction.CLUSTERSTATUS.toString(), new ZkNodeProps(props), rsp);
+ }
+
+ /**
+ * Handled list collection request.
+ * Do list collection request to zk host
+ *
+ * @param req solr request
+ * @param rsp solr response
+ * @throws KeeperException zk connection failed
+ * @throws InterruptedException connection interrupted
+ */
+ private void handleListAction(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
+ Map props = ZkNodeProps.makeMap(
+ Overseer.QUEUE_OPERATION, CollectionAction.LIST.toString().toLowerCase(Locale.ROOT));
+ handleResponse(CollectionAction.LIST.toString(), new ZkNodeProps(props), rsp);
+ }
+
+
public static ModifiableSolrParams params(String... params) {
ModifiableSolrParams msp = new ModifiableSolrParams();
for (int i=0; i filters = rb.getFilters();
+ if(filters != null) {
+ for(Query q : filters) {
+ if(q instanceof CollapsingQParserPlugin.CollapsingPostFilter) {
+ CollapsingQParserPlugin.CollapsingPostFilter cp = (CollapsingQParserPlugin.CollapsingPostFilter)q;
+ field = cp.getField();
+ }
+ }
+ }
+ }
+
+ if(field == null) {
+ throw new IOException("Expand field is null.");
+ }
+
String sortParam = params.get(ExpandParams.EXPAND_SORT);
+ String[] fqs = params.getParams(ExpandParams.EXPAND_FQ);
+ String qs = params.get(ExpandParams.EXPAND_Q);
int limit = params.getInt(ExpandParams.EXPAND_ROWS, 5);
Sort sort = null;
@@ -127,20 +145,40 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
sort = QueryParsing.parseSortSpec(sortParam, rb.req).getSort();
}
- Query query = rb.getQuery();
- List filters = rb.getFilters();
- List newFilters = new ArrayList();
- for(Query q : filters) {
- if(!(q instanceof CollapsingQParserPlugin.CollapsingPostFilter)) {
- newFilters.add(q);
- } else {
- CollapsingQParserPlugin.CollapsingPostFilter cp = (CollapsingQParserPlugin.CollapsingPostFilter)q;
- field = cp.getField();
+ Query query = null;
+ if(qs == null) {
+ query = rb.getQuery();
+ } else {
+ try {
+ QParser parser = QParser.getParser(qs, null, req);
+ query = parser.getQuery();
+ } catch(Exception e) {
+ throw new IOException(e);
}
}
- if(field == null) {
- throw new IOException("Expand field is null.");
+ List newFilters = new ArrayList();
+
+ if(fqs == null) {
+ List filters = rb.getFilters();
+ if(filters != null) {
+ for(Query q : filters) {
+ if(!(q instanceof CollapsingQParserPlugin.CollapsingPostFilter)) {
+ newFilters.add(q);
+ }
+ }
+ }
+ } else {
+ try {
+ for (String fq : fqs) {
+ if (fq != null && fq.trim().length()!=0 && !fq.equals("*:*")) {
+ QParser fqp = QParser.getParser(fq, null, req);
+ newFilters.add(fqp.getQuery());
+ }
+ }
+ } catch(Exception e) {
+ throw new IOException(e);
+ }
}
SolrIndexSearcher searcher = req.getSearcher();
diff --git a/solr/core/src/java/org/apache/solr/internal/csv/writer/CSVConfigGuesser.java b/solr/core/src/java/org/apache/solr/internal/csv/writer/CSVConfigGuesser.java
index 3f8096c63a4..50f598bac7f 100644
--- a/solr/core/src/java/org/apache/solr/internal/csv/writer/CSVConfigGuesser.java
+++ b/solr/core/src/java/org/apache/solr/internal/csv/writer/CSVConfigGuesser.java
@@ -21,8 +21,7 @@ package org.apache.solr.internal.csv.writer;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
-
-import org.apache.lucene.util.IOUtils;
+import java.nio.charset.StandardCharsets;
/**
* Tries to guess a config based on an InputStream.
@@ -76,7 +75,7 @@ public class CSVConfigGuesser {
public CSVConfig guess() {
try {
// tralalal
- BufferedReader bIn = new BufferedReader(new InputStreamReader(getInputStream(), IOUtils.CHARSET_UTF_8));
+ BufferedReader bIn = new BufferedReader(new InputStreamReader(getInputStream(), StandardCharsets.UTF_8));
String[] lines = new String[10];
String line = null;
int counter = 0;
diff --git a/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java b/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java
index e22f1da79c5..29fb474b178 100644
--- a/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java
+++ b/solr/core/src/java/org/apache/solr/rest/BaseSolrResource.java
@@ -45,13 +45,14 @@ import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.net.URLDecoder;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
/**
* Base class of all Solr Restlet server resource classes.
*/
public abstract class BaseSolrResource extends ServerResource {
- protected static final Charset UTF8 = Charset.forName("UTF-8");
+ protected static final Charset UTF8 = StandardCharsets.UTF_8;
protected static final String SHOW_DEFAULTS = "showDefaults";
private SolrCore solrCore;
@@ -161,7 +162,7 @@ public abstract class BaseSolrResource extends ServerResource {
binWriter.write(outputStream, solrRequest, solrResponse);
} else {
String charset = ContentStreamBase.getCharsetFromContentType(contentType);
- Writer out = (charset == null || charset.equalsIgnoreCase("UTF-8"))
+ Writer out = (charset == null)
? new OutputStreamWriter(outputStream, UTF8)
: new OutputStreamWriter(outputStream, charset);
out = new FastWriter(out);
diff --git a/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java b/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java
index e4218ca2348..e10b8a29c3e 100644
--- a/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java
+++ b/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java
@@ -29,6 +29,7 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
@@ -419,7 +420,7 @@ public abstract class ManagedResourceStorage {
public static final Logger log = LoggerFactory.getLogger(ManagedResourceStorage.class);
- public static final Charset UTF_8 = Charset.forName("UTF-8");
+ public static final Charset UTF_8 = StandardCharsets.UTF_8;
protected StorageIO storageIO;
protected SolrResourceLoader loader;
diff --git a/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymFilterFactory.java b/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymFilterFactory.java
new file mode 100644
index 00000000000..8a4bcbcec3d
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymFilterFactory.java
@@ -0,0 +1,349 @@
+package org.apache.solr.rest.schema.analysis;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+import java.text.ParseException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.synonym.SynonymFilterFactory;
+import org.apache.lucene.analysis.synonym.SynonymMap;
+import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.lucene.util.CharsRef;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.rest.BaseSolrResource;
+import org.apache.solr.rest.ManagedResource;
+import org.apache.solr.rest.ManagedResourceStorage.StorageIO;
+import org.restlet.data.Status;
+import org.restlet.resource.ResourceException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * TokenFilterFactory and ManagedResource implementation for
+ * doing CRUD on synonyms using the REST API.
+ */
+public class ManagedSynonymFilterFactory extends BaseManagedTokenFilterFactory {
+
+ public static final Logger log = LoggerFactory.getLogger(ManagedSynonymFilterFactory.class);
+
+ public static final String SYNONYM_MAPPINGS = "synonymMappings";
+ public static final String IGNORE_CASE_INIT_ARG = "ignoreCase";
+
+ /**
+ * ManagedResource implementation for synonyms, which are so specialized that
+ * it makes sense to implement this class as an inner class as it has little
+ * application outside the SynonymFilterFactory use cases.
+ */
+ public static class SynonymManager extends ManagedResource
+ implements ManagedResource.ChildResourceSupport
+ {
+
+ // TODO: Maybe hold this using a SoftReference / WeakReference to
+ // reduce memory in case the set of synonyms is large and the JVM
+ // is running low on memory?
+ protected Map> synonymMappings;
+
+ public SynonymManager(String resourceId, SolrResourceLoader loader, StorageIO storageIO)
+ throws SolrException {
+ super(resourceId, loader, storageIO);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ protected void onManagedDataLoadedFromStorage(NamedList managedInitArgs, Object managedData)
+ throws SolrException
+ {
+ NamedList initArgs = (NamedList)managedInitArgs;
+
+ String format = (String)initArgs.get("format");
+ if (format != null && !"solr".equals(format)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid format "+
+ format+"! Only 'solr' is supported.");
+ }
+
+ // the default behavior is to not ignore case,
+ // so if not supplied, then install the default
+ if (initArgs.get(IGNORE_CASE_INIT_ARG) == null) {
+ initArgs.add(IGNORE_CASE_INIT_ARG, Boolean.FALSE);
+ }
+ boolean ignoreCase = getIgnoreCase(managedInitArgs);
+ synonymMappings = new TreeMap<>();
+ if (managedData != null) {
+ Map storedSyns = (Map)managedData;
+ for (String key : storedSyns.keySet()) {
+ // give the nature of our JSON parsing solution, we really have
+ // no guarantees on what is in the file
+ Object mapping = storedSyns.get(key);
+ if (!(mapping instanceof List)) {
+ throw new SolrException(ErrorCode.SERVER_ERROR,
+ "Invalid synonym file format! Expected a list of synonyms for "+key+
+ " but got "+mapping.getClass().getName());
+ }
+
+ // if we're configured to ignoreCase, then we build the mappings with all lower
+ List vals = (List)storedSyns.get(key);
+ Set sortedVals = new TreeSet<>();
+ if (ignoreCase) {
+ for (String next : vals) {
+ sortedVals.add(applyCaseSetting(ignoreCase, next));
+ }
+ } else {
+ sortedVals.addAll(vals);
+ }
+
+ synonymMappings.put(applyCaseSetting(ignoreCase, key), sortedVals);
+ }
+ }
+
+ log.info("Loaded {} synonym mappings for {}", synonymMappings.size(), getResourceId());
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ protected Object applyUpdatesToManagedData(Object updates) {
+ if (!(updates instanceof Map)) {
+ throw new ResourceException(Status.CLIENT_ERROR_BAD_REQUEST,
+ "Unsupported data format (" + updates.getClass().getName() + "); expected a JSON object (Map)!");
+ }
+ boolean ignoreCase = getIgnoreCase();
+ boolean madeChanges = false;
+ Map jsonMap = (Map)updates;
+ for (String term : jsonMap.keySet()) {
+
+ term = applyCaseSetting(ignoreCase, term);
+
+ Set output = synonymMappings.get(term);
+
+ Object val = jsonMap.get(term);
+ if (val instanceof String) {
+ String strVal = applyCaseSetting(ignoreCase, (String)val);
+
+ if (output == null) {
+ output = new TreeSet<>();
+ synonymMappings.put(term, output);
+ }
+
+ if (output.add(strVal)) {
+ madeChanges = true;
+ }
+ } else if (val instanceof List) {
+ List vals = (List)val;
+
+ if (output == null) {
+ output = new TreeSet<>();
+ synonymMappings.put(term, output);
+ }
+
+ for (String nextVal : vals) {
+ if (output.add(applyCaseSetting(ignoreCase, nextVal))) {
+ madeChanges = true;
+ }
+ }
+
+ } else {
+ throw new ResourceException(Status.CLIENT_ERROR_BAD_REQUEST, "Unsupported value "+val+
+ " for "+term+"; expected single value or a JSON array!");
+ }
+ }
+
+ return madeChanges ? synonymMappings : null;
+ }
+
+ /**
+ * Handles a change in the ignoreCase setting for synonyms, which requires
+ * a full rebuild of the synonymMappings.
+ */
+ @Override
+ protected boolean updateInitArgs(NamedList updatedArgs) {
+ if (updatedArgs == null || updatedArgs.size() == 0) {
+ return false;
+ }
+ boolean currentIgnoreCase = getIgnoreCase(managedInitArgs);
+ boolean updatedIgnoreCase = getIgnoreCase(updatedArgs);
+ if (currentIgnoreCase == true && updatedIgnoreCase == false) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "Changing a managed word set's ignoreCase arg from true to false is not permitted.");
+ } else if (currentIgnoreCase == false && updatedIgnoreCase == true) {
+ // ignore case policy changed ... rebuild the map
+ Map> rebuild = new TreeMap<>();
+ for (String curr : synonymMappings.keySet()) {
+ Set newMappings = new TreeSet<>();
+ for (String next : synonymMappings.get(curr)) {
+ newMappings.add(applyCaseSetting(updatedIgnoreCase, next));
+ }
+ rebuild.put(applyCaseSetting(updatedIgnoreCase, curr), newMappings);
+ }
+ synonymMappings = rebuild;
+ }
+
+ return super.updateInitArgs(updatedArgs);
+ }
+
+ protected String applyCaseSetting(boolean ignoreCase, String str) {
+ return (ignoreCase && str != null) ? str.toLowerCase(Locale.ROOT) : str;
+ }
+
+ public boolean getIgnoreCase() {
+ return getIgnoreCase(managedInitArgs);
+ }
+
+ public boolean getIgnoreCase(NamedList initArgs) {
+ Boolean ignoreCase = initArgs.getBooleanArg(IGNORE_CASE_INIT_ARG);
+ // ignoreCase = false by default
+ return null == ignoreCase ? false : ignoreCase;
+ }
+
+ @Override
+ public void doGet(BaseSolrResource endpoint, String childId) {
+ SolrQueryResponse response = endpoint.getSolrResponse();
+ if (childId != null) {
+ boolean ignoreCase = getIgnoreCase();
+ String key = applyCaseSetting(ignoreCase, childId);
+ Set output = synonymMappings.get(key);
+ if (output == null) {
+ throw new SolrException(ErrorCode.NOT_FOUND,
+ String.format(Locale.ROOT, "%s not found in %s", key, getResourceId()));
+ }
+ response.add(key, output);
+ } else {
+ response.add(SYNONYM_MAPPINGS, buildMapToStore(synonymMappings));
+ }
+ }
+
+ @Override
+ public synchronized void doDeleteChild(BaseSolrResource endpoint, String childId) {
+ boolean ignoreCase = getIgnoreCase();
+ String key = applyCaseSetting(ignoreCase, childId);
+ Set output = synonymMappings.get(key);
+ if (output == null)
+ throw new SolrException(ErrorCode.NOT_FOUND,
+ String.format(Locale.ROOT, "%s not found in %s", key, getResourceId()));
+
+ synonymMappings.remove(key);
+ storeManagedData(synonymMappings);
+ log.info("Removed synonym mappings for: {}", key);
+ }
+ }
+
+ /**
+ * Custom SynonymMap.Parser implementation that provides synonym
+ * mappings from the managed JSON in this class during SynonymMap
+ * building.
+ */
+ private class ManagedSynonymParser extends SynonymMap.Parser {
+
+ SynonymManager synonymManager;
+
+ public ManagedSynonymParser(SynonymManager synonymManager, boolean dedup, Analyzer analyzer) {
+ super(dedup, analyzer);
+ this.synonymManager = synonymManager;
+ }
+
+ /**
+ * Add the managed synonyms and their mappings into the SynonymMap builder.
+ */
+ @Override
+ public void parse(Reader in) throws IOException, ParseException {
+ for (String term : synonymManager.synonymMappings.keySet()) {
+ for (String mapping : synonymManager.synonymMappings.get(term)) {
+ add(new CharsRef(term), new CharsRef(mapping), false);
+ }
+ }
+ }
+ }
+
+ protected SynonymFilterFactory delegate;
+
+ public ManagedSynonymFilterFactory(Map args) {
+ super(args);
+ }
+
+ @Override
+ public String getResourceId() {
+ return "/schema/analysis/synonyms/"+handle;
+ }
+
+ protected Class getManagedResourceImplClass() {
+ return SynonymManager.class;
+ }
+
+ /**
+ * Called once, during core initialization, to initialize any analysis components
+ * that depend on the data managed by this resource. It is important that the
+ * analysis component is only initialized once during core initialization so that
+ * text analysis is consistent, especially in a distributed environment, as we
+ * don't want one server applying a different set of stop words than other servers.
+ */
+ @SuppressWarnings("unchecked")
+ @Override
+ public void onManagedResourceInitialized(NamedList initArgs, final ManagedResource res)
+ throws SolrException
+ {
+ NamedList args = (NamedList)initArgs;
+ args.add("synonyms", getResourceId());
+ args.add("expand", "false");
+ args.add("format", "solr");
+
+ Map filtArgs = new HashMap<>();
+ for (Map.Entry entry : args) {
+ filtArgs.put(entry.getKey(), entry.getValue().toString());
+ }
+ // create the actual filter factory that pulls the synonym mappings
+ // from synonymMappings using a custom parser implementation
+ delegate = new SynonymFilterFactory(filtArgs) {
+ @Override
+ protected SynonymMap loadSynonyms
+ (ResourceLoader loader, String cname, boolean dedup, Analyzer analyzer)
+ throws IOException, ParseException {
+
+ ManagedSynonymParser parser =
+ new ManagedSynonymParser((SynonymManager)res, dedup, analyzer);
+ // null is safe here because there's no actual parsing done against a input Reader
+ parser.parse(null);
+ return parser.build();
+ }
+ };
+ try {
+ delegate.inform(res.getResourceLoader());
+ } catch (IOException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
+ }
+
+ @Override
+ public TokenStream create(TokenStream input) {
+ if (delegate == null)
+ throw new IllegalStateException(this.getClass().getName()+
+ " not initialized correctly! The SynonymFilterFactory delegate was not initialized.");
+
+ return delegate.create(input);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/schema/BCDIntField.java b/solr/core/src/java/org/apache/solr/schema/BCDIntField.java
index 86efdf11889..17679b67b72 100644
--- a/solr/core/src/java/org/apache/solr/schema/BCDIntField.java
+++ b/solr/core/src/java/org/apache/solr/schema/BCDIntField.java
@@ -66,6 +66,16 @@ public class BCDIntField extends PrimitiveFieldType {
public void write(TextResponseWriter writer, String name, StorableField f) throws IOException {
writer.writeInt(name,toExternal(f));
}
+
+ @Override
+ public Object marshalSortValue(Object value) {
+ return marshalStringSortValue(value);
+ }
+
+ @Override
+ public Object unmarshalSortValue(Object value) {
+ return unmarshalStringSortValue(value);
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/schema/BoolField.java b/solr/core/src/java/org/apache/solr/schema/BoolField.java
index 242de06a314..07f5089591f 100644
--- a/solr/core/src/java/org/apache/solr/schema/BoolField.java
+++ b/solr/core/src/java/org/apache/solr/schema/BoolField.java
@@ -151,6 +151,16 @@ public class BoolField extends PrimitiveFieldType {
public void write(TextResponseWriter writer, String name, StorableField f) throws IOException {
writer.writeBool(name, f.stringValue().charAt(0) == 'T');
}
+
+ @Override
+ public Object marshalSortValue(Object value) {
+ return marshalStringSortValue(value);
+ }
+
+ @Override
+ public Object unmarshalSortValue(Object value) {
+ return unmarshalStringSortValue(value);
+ }
}
// TODO - this can be much more efficient - use OpenBitSet or Bits
diff --git a/solr/core/src/java/org/apache/solr/schema/CollationField.java b/solr/core/src/java/org/apache/solr/schema/CollationField.java
index 13f35a8c069..bf0dc1dbfdc 100644
--- a/solr/core/src/java/org/apache/solr/schema/CollationField.java
+++ b/solr/core/src/java/org/apache/solr/schema/CollationField.java
@@ -47,7 +47,6 @@ import org.apache.lucene.util.Version;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.util.Base64;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
@@ -278,20 +277,11 @@ public class CollationField extends FieldType {
@Override
public Object marshalSortValue(Object value) {
- if (null == value) {
- return null;
- }
- final BytesRef val = (BytesRef)value;
- return Base64.byteArrayToBase64(val.bytes, val.offset, val.length);
+ return marshalBase64SortValue(value);
}
@Override
public Object unmarshalSortValue(Object value) {
- if (null == value) {
- return null;
- }
- final String val = (String)value;
- final byte[] bytes = Base64.base64ToByteArray(val);
- return new BytesRef(bytes);
+ return unmarshalBase64SortValue(value);
}
}
diff --git a/solr/core/src/java/org/apache/solr/schema/DateField.java b/solr/core/src/java/org/apache/solr/schema/DateField.java
index faf9c4ff93f..52eac81f8f0 100644
--- a/solr/core/src/java/org/apache/solr/schema/DateField.java
+++ b/solr/core/src/java/org/apache/solr/schema/DateField.java
@@ -247,6 +247,16 @@ public class DateField extends PrimitiveFieldType implements DateValueFieldType
return getStringSort(field,reverse);
}
+ @Override
+ public Object marshalSortValue(Object value) {
+ return marshalStringSortValue(value);
+ }
+
+ @Override
+ public Object unmarshalSortValue(Object value) {
+ return unmarshalStringSortValue(value);
+ }
+
@Override
public void write(TextResponseWriter writer, String name, StorableField f) throws IOException {
writer.writeDate(name, toExternal(f));
diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java
index 4d4ae5a225d..32c7a63bb8f 100644
--- a/solr/core/src/java/org/apache/solr/schema/FieldType.java
+++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java
@@ -48,6 +48,7 @@ import org.apache.solr.analysis.SolrAnalyzer;
import org.apache.solr.analysis.TokenizerChain;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.util.Base64;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.response.TextResponseWriter;
@@ -968,4 +969,52 @@ public abstract class FieldType extends FieldProperties {
public Object unmarshalSortValue(Object value) {
return value;
}
+
+ /**
+ * Marshals a string-based field value.
+ */
+ protected static Object marshalStringSortValue(Object value) {
+ if (null == value) {
+ return null;
+ }
+ CharsRef spare = new CharsRef();
+ UnicodeUtil.UTF8toUTF16((BytesRef)value, spare);
+ return spare.toString();
+ }
+
+ /**
+ * Unmarshals a string-based field value.
+ */
+ protected static Object unmarshalStringSortValue(Object value) {
+ if (null == value) {
+ return null;
+ }
+ BytesRef spare = new BytesRef();
+ String stringVal = (String)value;
+ UnicodeUtil.UTF16toUTF8(stringVal, 0, stringVal.length(), spare);
+ return spare;
+ }
+
+ /**
+ * Marshals a binary field value.
+ */
+ protected static Object marshalBase64SortValue(Object value) {
+ if (null == value) {
+ return null;
+ }
+ final BytesRef val = (BytesRef)value;
+ return Base64.byteArrayToBase64(val.bytes, val.offset, val.length);
+ }
+
+ /**
+ * Unmarshals a binary field value.
+ */
+ protected static Object unmarshalBase64SortValue(Object value) {
+ if (null == value) {
+ return null;
+ }
+ final String val = (String)value;
+ final byte[] bytes = Base64.base64ToByteArray(val);
+ return new BytesRef(bytes);
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/schema/FloatField.java b/solr/core/src/java/org/apache/solr/schema/FloatField.java
index 7e23443852d..6d8053580fa 100644
--- a/solr/core/src/java/org/apache/solr/schema/FloatField.java
+++ b/solr/core/src/java/org/apache/solr/schema/FloatField.java
@@ -70,7 +70,7 @@ public class FloatField extends PrimitiveFieldType implements FloatValueFieldTyp
@Override
public SortField getSortField(SchemaField field,boolean reverse) {
field.checkSortability();
- return new SortField(field.name,SortField.Type.FLOAT, reverse);
+ return new SortField(field.name, PARSER, reverse);
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
index fc44e39c32a..43a312744b3 100644
--- a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
+++ b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
@@ -33,11 +33,13 @@ import org.w3c.dom.Document;
import org.xml.sax.InputSource;
import javax.xml.xpath.XPath;
+
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -92,7 +94,7 @@ public final class ManagedIndexSchema extends IndexSchema {
}
}
final FileOutputStream out = new FileOutputStream(managedSchemaFile);
- writer = new OutputStreamWriter(out, "UTF-8");
+ writer = new OutputStreamWriter(out, StandardCharsets.UTF_8);
persist(writer);
log.info("Upgraded to managed schema at " + managedSchemaFile.getPath());
} catch (IOException e) {
@@ -132,7 +134,7 @@ public final class ManagedIndexSchema extends IndexSchema {
StringWriter writer = new StringWriter();
persist(writer);
- final byte[] data = writer.toString().getBytes("UTF-8");
+ final byte[] data = writer.toString().getBytes(StandardCharsets.UTF_8);
if (createOnly) {
try {
zkClient.create(managedSchemaPath, data, CreateMode.PERSISTENT, true);
diff --git a/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java b/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java
index c82266c391b..e6927cefef4 100644
--- a/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java
+++ b/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java
@@ -20,13 +20,13 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.noggit.JSONParser;
import org.apache.lucene.analysis.util.ResourceLoader;
-import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.slf4j.Logger;
@@ -202,7 +202,7 @@ public class OpenExchangeRatesOrgProvider implements ExchangeRateProvider {
private JSONParser parser;
public OpenExchangeRates(InputStream ratesStream) throws IOException {
- parser = new JSONParser(new InputStreamReader(ratesStream, IOUtils.CHARSET_UTF_8));
+ parser = new JSONParser(new InputStreamReader(ratesStream, StandardCharsets.UTF_8));
rates = new HashMap<>();
int ev;
diff --git a/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java b/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
index 3e495747730..966e0954749 100644
--- a/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
+++ b/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
@@ -78,6 +78,7 @@ public class PreAnalyzedField extends FieldType {
parser = new JsonPreAnalyzedParser();
}
}
+ args.remove(PARSER_IMPL);
}
}
diff --git a/solr/core/src/java/org/apache/solr/schema/StrField.java b/solr/core/src/java/org/apache/solr/schema/StrField.java
index e39d7e6ad64..9fc4320eda1 100644
--- a/solr/core/src/java/org/apache/solr/schema/StrField.java
+++ b/solr/core/src/java/org/apache/solr/schema/StrField.java
@@ -29,8 +29,6 @@ import org.apache.lucene.index.StorableField;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.CharsRef;
-import org.apache.lucene.util.UnicodeUtil;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
@@ -86,23 +84,12 @@ public class StrField extends PrimitiveFieldType {
@Override
public Object marshalSortValue(Object value) {
- if (null == value) {
- return null;
- }
- CharsRef spare = new CharsRef();
- UnicodeUtil.UTF8toUTF16((BytesRef)value, spare);
- return spare.toString();
+ return marshalStringSortValue(value);
}
@Override
public Object unmarshalSortValue(Object value) {
- if (null == value) {
- return null;
- }
- BytesRef spare = new BytesRef();
- String stringVal = (String)value;
- UnicodeUtil.UTF16toUTF8(stringVal, 0, stringVal.length(), spare);
- return spare;
+ return unmarshalStringSortValue(value);
}
}
diff --git a/solr/core/src/java/org/apache/solr/schema/TextField.java b/solr/core/src/java/org/apache/solr/schema/TextField.java
index f0741f51445..68c740dbc57 100644
--- a/solr/core/src/java/org/apache/solr/schema/TextField.java
+++ b/solr/core/src/java/org/apache/solr/schema/TextField.java
@@ -23,9 +23,7 @@ import org.apache.lucene.index.StorableField;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.QueryBuilder;
-import org.apache.lucene.util.UnicodeUtil;
import org.apache.solr.common.SolrException;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
@@ -170,22 +168,11 @@ public class TextField extends FieldType {
@Override
public Object marshalSortValue(Object value) {
- if (null == value) {
- return null;
- }
- CharsRef spare = new CharsRef();
- UnicodeUtil.UTF8toUTF16((BytesRef)value, spare);
- return spare.toString();
+ return marshalStringSortValue(value);
}
@Override
public Object unmarshalSortValue(Object value) {
- if (null == value) {
- return null;
- }
- BytesRef spare = new BytesRef();
- String stringVal = (String)value;
- UnicodeUtil.UTF16toUTF8(stringVal, 0, stringVal.length(), spare);
- return spare;
+ return unmarshalStringSortValue(value);
}
}
diff --git a/solr/core/src/java/org/apache/solr/schema/TrieDateField.java b/solr/core/src/java/org/apache/solr/schema/TrieDateField.java
index 0a652efb44c..e92d601f437 100644
--- a/solr/core/src/java/org/apache/solr/schema/TrieDateField.java
+++ b/solr/core/src/java/org/apache/solr/schema/TrieDateField.java
@@ -83,6 +83,16 @@ public class TrieDateField extends DateField implements DateValueFieldType {
return wrappedField.getSortField(field, top);
}
+ @Override
+ public Object marshalSortValue(Object value) {
+ return value;
+ }
+
+ @Override
+ public Object unmarshalSortValue(Object value) {
+ return value;
+ }
+
@Override
public ValueSource getValueSource(SchemaField field, QParser parser) {
return wrappedField.getValueSource(field, parser);
diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
index 0f6d9e57c08..601790c41f5 100644
--- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
@@ -443,6 +443,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private int nullDoc;
private FloatArrayList nullScores;
private IntOpenHashSet boostDocs;
+ private int[] boostOrds;
public CollapsingScoreCollector(int maxDoc,
int segments,
@@ -455,11 +456,19 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.boostDocs = boostDocs;
if(this.boostDocs != null) {
//Set the elevated docs now.
+ IntOpenHashSet boostG = new IntOpenHashSet();
Iterator it = this.boostDocs.iterator();
while(it.hasNext()) {
IntCursor cursor = it.next();
- this.collapsedSet.set(cursor.value);
+ int i = cursor.value;
+ this.collapsedSet.set(i);
+ int ord = values.getOrd(i);
+ if(ord > -1) {
+ boostG.add(ord);
+ }
}
+ boostOrds = boostG.toArray();
+ Arrays.sort(boostOrds);
}
this.values = values;
int valueCount = values.getValueCount();
@@ -489,6 +498,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
public void collect(int docId) throws IOException {
int globalDoc = docId+this.docBase;
int ord = values.getOrd(globalDoc);
+
if(ord > -1) {
float score = scorer.score();
if(score > scores[ord]) {
@@ -520,6 +530,12 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.collapsedSet.set(nullDoc);
}
+ if(this.boostOrds != null) {
+ for(int i=0; i -1) {
@@ -539,6 +555,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
while((docId = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
int ord = values.getOrd(docId);
+
if(ord > -1) {
dummy.score = scores[ord];
} else if(this.boostDocs != null && boostDocs.contains(docId)) {
@@ -600,14 +617,14 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.needsScores = needsScores;
this.boostDocs = boostDocs;
if(funcQuery != null) {
- this.fieldValueCollapse = new ValueSourceCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs, funcQuery, searcher);
+ this.fieldValueCollapse = new ValueSourceCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs, funcQuery, searcher, values);
} else {
if(fieldType instanceof TrieIntField) {
- this.fieldValueCollapse = new IntValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
+ this.fieldValueCollapse = new IntValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs, values);
} else if(fieldType instanceof TrieLongField) {
- this.fieldValueCollapse = new LongValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
+ this.fieldValueCollapse = new LongValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs, values);
} else if(fieldType instanceof TrieFloatField) {
- this.fieldValueCollapse = new FloatValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs);
+ this.fieldValueCollapse = new FloatValueCollapse(maxDoc, field, nullPolicy, new int[valueCount], max, this.needsScores, boostDocs, values);
} else {
throw new IOException("min/max must be either TrieInt, TrieLong or TrieFloat.");
}
@@ -696,6 +713,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
protected float[] scores;
protected FixedBitSet collapsedSet;
protected IntOpenHashSet boostDocs;
+ protected int[] boostOrds;
protected int nullDoc = -1;
protected boolean needsScores;
protected boolean max;
@@ -709,7 +727,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int nullPolicy,
boolean max,
boolean needsScores,
- IntOpenHashSet boostDocs) {
+ IntOpenHashSet boostDocs,
+ SortedDocValues values) {
this.field = field;
this.nullPolicy = nullPolicy;
this.max = max;
@@ -717,11 +736,19 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.collapsedSet = new FixedBitSet(maxDoc);
this.boostDocs = boostDocs;
if(this.boostDocs != null) {
+ IntOpenHashSet boostG = new IntOpenHashSet();
Iterator it = boostDocs.iterator();
while(it.hasNext()) {
IntCursor cursor = it.next();
- this.collapsedSet.set(cursor.value);
+ int i = cursor.value;
+ this.collapsedSet.set(i);
+ int ord = values.getOrd(i);
+ if(ord > -1) {
+ boostG.add(ord);
+ }
}
+ this.boostOrds = boostG.toArray();
+ Arrays.sort(this.boostOrds);
}
}
@@ -730,6 +757,12 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.collapsedSet.set(nullDoc);
}
+ if(this.boostOrds != null) {
+ for(int i=0; i -1) {
@@ -770,8 +803,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int[] ords,
boolean max,
boolean needsScores,
- IntOpenHashSet boostDocs) throws IOException {
- super(maxDoc, field, nullPolicy, max, needsScores, boostDocs);
+ IntOpenHashSet boostDocs, SortedDocValues values) throws IOException {
+ super(maxDoc, field, nullPolicy, max, needsScores, boostDocs, values);
this.ords = ords;
this.ordVals = new int[ords.length];
Arrays.fill(ords, -1);
@@ -838,8 +871,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int[] ords,
boolean max,
boolean needsScores,
- IntOpenHashSet boostDocs) throws IOException {
- super(maxDoc, field, nullPolicy, max, needsScores, boostDocs);
+ IntOpenHashSet boostDocs, SortedDocValues values) throws IOException {
+ super(maxDoc, field, nullPolicy, max, needsScores, boostDocs, values);
this.ords = ords;
this.ordVals = new long[ords.length];
Arrays.fill(ords, -1);
@@ -907,8 +940,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int[] ords,
boolean max,
boolean needsScores,
- IntOpenHashSet boostDocs) throws IOException {
- super(maxDoc, field, nullPolicy, max, needsScores, boostDocs);
+ IntOpenHashSet boostDocs, SortedDocValues values) throws IOException {
+ super(maxDoc, field, nullPolicy, max, needsScores, boostDocs, values);
this.ords = ords;
this.ordVals = new float[ords.length];
Arrays.fill(ords, -1);
@@ -982,8 +1015,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
boolean max,
boolean needsScores,
IntOpenHashSet boostDocs,
- FunctionQuery funcQuery, IndexSearcher searcher) throws IOException {
- super(maxDoc, null, nullPolicy, max, needsScores, boostDocs);
+ FunctionQuery funcQuery, IndexSearcher searcher, SortedDocValues values) throws IOException {
+ super(maxDoc, null, nullPolicy, max, needsScores, boostDocs, values);
this.valueSource = funcQuery.getValueSource();
this.rcontext = ValueSource.newContext(searcher);
this.ords = ords;
diff --git a/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java b/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java
index a2c1acc2e75..0f8c14c40a6 100644
--- a/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java
+++ b/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java
@@ -22,7 +22,6 @@ import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.FloatDocValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.handler.RequestHandlerUtils;
@@ -39,6 +38,7 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.util.*;
/**
@@ -252,7 +252,7 @@ public class FileFloatSource extends ValueSource {
return vals;
}
- BufferedReader r = new BufferedReader(new InputStreamReader(is, IOUtils.CHARSET_UTF_8));
+ BufferedReader r = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
String idName = ffs.keyField.getName();
FieldType idType = ffs.keyField.getType();
diff --git a/solr/core/src/java/org/apache/solr/servlet/BaseSolrFilter.java b/solr/core/src/java/org/apache/solr/servlet/BaseSolrFilter.java
new file mode 100644
index 00000000000..966bad2916f
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/servlet/BaseSolrFilter.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.servlet;
+
+import javax.servlet.Filter;
+
+/**
+ * All Solr filters available to the user's webapp should
+ * extend this class and not just implement {@link Filter}.
+ * This class ensures that the logging configuration is correct
+ * before any Solr specific code is executed.
+ */
+abstract class BaseSolrFilter implements Filter {
+
+ static {
+ CheckLoggingConfiguration.check();
+ }
+
+}
diff --git a/solr/core/src/java/org/apache/solr/servlet/BaseSolrServlet.java b/solr/core/src/java/org/apache/solr/servlet/BaseSolrServlet.java
new file mode 100644
index 00000000000..3a1f88ac333
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/servlet/BaseSolrServlet.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.servlet;
+
+import javax.servlet.http.HttpServlet;
+
+/**
+ * All Solr servlets available to the user's webapp should
+ * extend this class and not {@link HttpServlet}.
+ * This class ensures that the logging configuration is correct
+ * before any Solr specific code is executed.
+ */
+@SuppressWarnings("serial")
+abstract class BaseSolrServlet extends HttpServlet {
+
+ static {
+ CheckLoggingConfiguration.check();
+ }
+
+}
diff --git a/solr/core/src/java/org/apache/solr/servlet/CheckLoggingConfiguration.java b/solr/core/src/java/org/apache/solr/servlet/CheckLoggingConfiguration.java
new file mode 100644
index 00000000000..bd8842cfc74
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/servlet/CheckLoggingConfiguration.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.servlet;
+
+import org.slf4j.LoggerFactory;
+
+final class CheckLoggingConfiguration {
+
+ static void check() {
+ try {
+ LoggerFactory.getLogger(CheckLoggingConfiguration.class);
+ } catch (NoClassDefFoundError e) {
+ throw new NoClassDefFoundError("Failed to initialize Apache Solr: "
+ +"Could not find necessary SLF4j logging jars. If using Jetty, the SLF4j logging jars need to go in "
+ +"the jetty lib/ext directory. For other containers, the corresponding directory should be used. "
+ +"For more information, see: http://wiki.apache.org/solr/SolrLogging");
+ }
+ }
+
+ private CheckLoggingConfiguration() {}
+
+}
diff --git a/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java b/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java
index beb95aa92a4..157209737dc 100644
--- a/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java
+++ b/solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java
@@ -21,8 +21,8 @@ import java.io.InputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.nio.charset.StandardCharsets;
-import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -37,7 +37,7 @@ import org.apache.solr.core.SolrCore;
*
* @since solr 4.0
*/
-public final class LoadAdminUiServlet extends HttpServlet {
+public final class LoadAdminUiServlet extends BaseSolrServlet {
@Override
public void doGet(HttpServletRequest request,
@@ -51,7 +51,7 @@ public final class LoadAdminUiServlet extends HttpServlet {
try {
response.setCharacterEncoding("UTF-8");
response.setContentType("text/html");
- Writer out = new OutputStreamWriter(response.getOutputStream(), "UTF-8");
+ Writer out = new OutputStreamWriter(response.getOutputStream(), StandardCharsets.UTF_8);
String html = IOUtils.toString(in, "UTF-8");
Package pack = SolrCore.class.getPackage();
diff --git a/solr/core/src/java/org/apache/solr/servlet/RedirectServlet.java b/solr/core/src/java/org/apache/solr/servlet/RedirectServlet.java
index bc497461760..4661f82b3c6 100644
--- a/solr/core/src/java/org/apache/solr/servlet/RedirectServlet.java
+++ b/solr/core/src/java/org/apache/solr/servlet/RedirectServlet.java
@@ -21,14 +21,13 @@ import java.io.IOException;
import javax.servlet.ServletConfig;
import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
/**
* A Simple redirection servlet to help us deprecate old UI elements
*/
-public class RedirectServlet extends HttpServlet{
+public class RedirectServlet extends BaseSolrServlet {
static final String CONTEXT_KEY = "${context}";
diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
index ad1fa3755b1..efee05ea3c0 100644
--- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
+++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
@@ -73,7 +73,6 @@ import org.apache.solr.util.FastWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.servlet.Filter;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
import javax.servlet.ServletException;
@@ -81,6 +80,7 @@ import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -89,6 +89,7 @@ import java.io.OutputStreamWriter;
import java.io.Writer;
import java.net.URL;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -105,13 +106,12 @@ import java.util.Set;
*
* @since solr 1.2
*/
-public class SolrDispatchFilter implements Filter
-{
+public class SolrDispatchFilter extends BaseSolrFilter {
private static final String CONNECTION_HEADER = "Connection";
private static final String TRANSFER_ENCODING_HEADER = "Transfer-Encoding";
private static final String CONTENT_LENGTH_HEADER = "Content-Length";
- final Logger log;
+ static final Logger log = LoggerFactory.getLogger(SolrDispatchFilter.class);
protected volatile CoreContainer cores;
@@ -119,19 +119,9 @@ public class SolrDispatchFilter implements Filter
protected String abortErrorMessage = null;
protected final HttpClient httpClient = HttpClientUtil.createClient(new ModifiableSolrParams());
- private static final Charset UTF8 = Charset.forName("UTF-8");
+ private static final Charset UTF8 = StandardCharsets.UTF_8;
public SolrDispatchFilter() {
- try {
- log = LoggerFactory.getLogger(SolrDispatchFilter.class);
- } catch (NoClassDefFoundError e) {
- throw new SolrException(
- ErrorCode.SERVER_ERROR,
- "Could not find necessary SLF4j logging jars. If using Jetty, the SLF4j logging jars need to go in "
- +"the jetty lib/ext directory. For other containers, the corresponding directory should be used. "
- +"For more information, see: http://wiki.apache.org/solr/SolrLogging",
- e);
- }
}
@Override
@@ -765,7 +755,7 @@ public class SolrDispatchFilter implements Filter
binWriter.write(response.getOutputStream(), solrReq, solrRsp);
} else {
String charset = ContentStreamBase.getCharsetFromContentType(ct);
- Writer out = (charset == null || charset.equalsIgnoreCase("UTF-8"))
+ Writer out = (charset == null)
? new OutputStreamWriter(response.getOutputStream(), UTF8)
: new OutputStreamWriter(response.getOutputStream(), charset);
out = new FastWriter(out);
diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
index f4eccf47cd0..f3fa91d78c8 100644
--- a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
+++ b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
@@ -27,6 +27,7 @@ import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -244,7 +245,7 @@ public class SolrRequestParsers
}
}
};
- parseFormDataContent(in, Long.MAX_VALUE, IOUtils.CHARSET_UTF_8, map, true);
+ parseFormDataContent(in, Long.MAX_VALUE, StandardCharsets.UTF_8, map, true);
} catch (IOException ioe) {
throw new SolrException(ErrorCode.BAD_REQUEST, ioe);
}
@@ -598,7 +599,7 @@ public class SolrRequestParsers
// get query String from request body, using the charset given in content-type:
final String cs = ContentStreamBase.getCharsetFromContentType(req.getContentType());
- final Charset charset = (cs == null) ? IOUtils.CHARSET_UTF_8 : Charset.forName(cs);
+ final Charset charset = (cs == null) ? StandardCharsets.UTF_8 : Charset.forName(cs);
InputStream in = null;
try {
in = req.getInputStream();
diff --git a/solr/core/src/java/org/apache/solr/servlet/ZookeeperInfoServlet.java b/solr/core/src/java/org/apache/solr/servlet/ZookeeperInfoServlet.java
index 917b5cd9c44..06718c32410 100644
--- a/solr/core/src/java/org/apache/solr/servlet/ZookeeperInfoServlet.java
+++ b/solr/core/src/java/org/apache/solr/servlet/ZookeeperInfoServlet.java
@@ -18,22 +18,18 @@
package org.apache.solr.servlet;
import java.io.IOException;
-import java.io.BufferedWriter;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.util.Date;
import java.util.List;
import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-import org.noggit.CharArr;
-import org.noggit.JSONWriter;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.SolrZkClient;
@@ -42,6 +38,8 @@ import org.apache.solr.core.CoreContainer;
import org.apache.solr.util.FastWriter;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
+import org.noggit.CharArr;
+import org.noggit.JSONWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,7 +49,7 @@ import org.slf4j.LoggerFactory;
*
* @since solr 4.0
*/
-public final class ZookeeperInfoServlet extends HttpServlet {
+public final class ZookeeperInfoServlet extends BaseSolrServlet {
static final Logger log = LoggerFactory.getLogger(ZookeeperInfoServlet.class);
@Override
@@ -96,7 +94,7 @@ public final class ZookeeperInfoServlet extends HttpServlet {
response.setCharacterEncoding("UTF-8");
response.setContentType("application/json");
- Writer out = new FastWriter(new OutputStreamWriter(response.getOutputStream(), IOUtils.CHARSET_UTF_8));
+ Writer out = new FastWriter(new OutputStreamWriter(response.getOutputStream(), StandardCharsets.UTF_8));
ZKPrinter printer = new ZKPrinter(response, out, cores.getZkController(), addr);
printer.detail = detail;
diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/FileDictionaryFactory.java b/solr/core/src/java/org/apache/solr/spelling/suggest/FileDictionaryFactory.java
index 986fe7ec64e..07ecb4334c2 100644
--- a/solr/core/src/java/org/apache/solr/spelling/suggest/FileDictionaryFactory.java
+++ b/solr/core/src/java/org/apache/solr/spelling/suggest/FileDictionaryFactory.java
@@ -19,10 +19,10 @@ package org.apache.solr.spelling.suggest;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import org.apache.lucene.search.spell.Dictionary;
import org.apache.lucene.search.suggest.FileDictionary;
-import org.apache.lucene.util.IOUtils;
import org.apache.solr.core.SolrCore;
import org.apache.solr.search.SolrIndexSearcher;
@@ -53,7 +53,7 @@ public class FileDictionaryFactory extends DictionaryFactory {
try {
return new FileDictionary(new InputStreamReader(
- core.getResourceLoader().openResource(sourceLocation), IOUtils.CHARSET_UTF_8), fieldDelimiter);
+ core.getResourceLoader().openResource(sourceLocation), StandardCharsets.UTF_8), fieldDelimiter);
} catch (IOException e) {
throw new RuntimeException();
}
diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java b/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java
index 521bb94398f..8c1293a3680 100644
--- a/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java
+++ b/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java
@@ -23,6 +23,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.List;
@@ -131,7 +132,7 @@ public class Suggester extends SolrSpellChecker {
} else {
try {
dictionary = new FileDictionary(new InputStreamReader(
- core.getResourceLoader().openResource(sourceLocation), IOUtils.CHARSET_UTF_8));
+ core.getResourceLoader().openResource(sourceLocation), StandardCharsets.UTF_8));
} catch (UnsupportedEncodingException e) {
// should not happen
LOG.error("should not happen", e);
diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FreeTextLookupFactory.java b/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FreeTextLookupFactory.java
index 2ec64523753..730a2ba417d 100644
--- a/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FreeTextLookupFactory.java
+++ b/solr/core/src/java/org/apache/solr/spelling/suggest/fst/FreeTextLookupFactory.java
@@ -1,9 +1,10 @@
package org.apache.solr.spelling.suggest.fst;
+import java.nio.charset.StandardCharsets;
+
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.analyzing.FreeTextSuggester;
-import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.FieldType;
@@ -71,7 +72,7 @@ public class FreeTextLookupFactory extends LookupFactory {
: FreeTextSuggester.DEFAULT_GRAMS;
byte separator = (params.get(SEPARATOR) != null)
- ? params.get(SEPARATOR).toString().getBytes(IOUtils.CHARSET_UTF_8)[0]
+ ? params.get(SEPARATOR).toString().getBytes(StandardCharsets.UTF_8)[0]
: FreeTextSuggester.DEFAULT_SEPARATOR;
return new FreeTextSuggester(indexAnalyzer, queryAnalyzer, grams, separator);
diff --git a/solr/core/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java b/solr/core/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java
index 858214cf83b..5c76a9848ac 100644
--- a/solr/core/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java
+++ b/solr/core/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java
@@ -88,5 +88,10 @@ public class CachedIndexOutput extends ReusedBufferedIndexOutput {
offset += len;
}
}
-
+
+ @Override
+ public long getChecksum() throws IOException {
+ flush();
+ return dest.getChecksum();
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java b/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java
index 942dfd73f4f..605d2700955 100644
--- a/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java
+++ b/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java
@@ -66,5 +66,9 @@ public class NullIndexOutput extends IndexOutput {
length = pos;
}
}
-
+
+ @Override
+ public long getChecksum() throws IOException {
+ return 0; // we don't write anything.
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
index 52435398c98..f4d1fcc0954 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.net.ConnectException;
import java.util.ArrayList;
import java.util.List;
-
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
@@ -206,7 +205,7 @@ public class SolrCmdDistributor {
void addCommit(UpdateRequest ureq, CommitUpdateCommand cmd) {
if (cmd == null) return;
ureq.setAction(cmd.optimize ? AbstractUpdateRequest.ACTION.OPTIMIZE
- : AbstractUpdateRequest.ACTION.COMMIT, false, cmd.waitSearcher, cmd.maxOptimizeSegments, cmd.softCommit, cmd.expungeDeletes);
+ : AbstractUpdateRequest.ACTION.COMMIT, false, cmd.waitSearcher, cmd.maxOptimizeSegments, cmd.softCommit, cmd.expungeDeletes, cmd.openSearcher);
}
private void submit(Req req) {
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DocExpirationUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/DocExpirationUpdateProcessorFactory.java
new file mode 100644
index 00000000000..a006ebbb257
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/update/processor/DocExpirationUpdateProcessorFactory.java
@@ -0,0 +1,510 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update.processor;
+
+import java.io.IOException;
+
+import org.apache.solr.common.SolrException;
+import static org.apache.solr.common.SolrException.ErrorCode.*;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+
+import org.apache.solr.core.CloseHook;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.cloud.CloudDescriptor;
+import org.apache.solr.cloud.ZkController;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.DateField;
+import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.update.CommitUpdateCommand;
+import org.apache.solr.update.DeleteUpdateCommand;
+import org.apache.solr.util.DateMathParser;
+import org.apache.solr.util.DefaultSolrThreadFactory;
+import org.apache.solr.util.plugin.SolrCoreAware;
+
+import java.text.ParseException;
+import java.util.Comparator;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Collections;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * Update Processor Factory for managing automatic "expiration" of documents.
+ *
+ *
+ *
+ * The DocExpirationUpdateProcessorFactory provides two features related
+ * to the "expiration" of documents which can be used individually, or in combination:
+ *
+ *
+ * Computing expiration field values for documents from a "time to live" (TTL)
+ * Periodically delete documents from the index based on an expiration field
+ *
+ *
+ *
+ * Documents with expiration field values computed from a TTL can be be excluded from
+ * searchers using simple date based filters relative to NOW, or completely
+ * removed from the index using the periodic delete function of this factory. Alternatively,
+ * the periodic delete function of this factory can be used to remove any document with an
+ * expiration value - even if that expiration was explicitly set with-out leveraging the TTL
+ * feature of this factory.
+ *
+ *
+ *
+ * The following configuration options are supported:
+ *
+ *
+ *
+ * expirationFieldName - The name of the expiration field to use
+ * in any operations (mandatory).
+ * ttlFieldName - Name of a field this process should look
+ * for in each document processed, defaulting to _ttl_.
+ * If the specified field name exists in a document, the document field value
+ * will be parsed as a {@linkplain DateMathParser Date Math Expression} relative to
+ * NOW and the result will be added to the document using the
+ * expirationFieldName. Use <null name="ttlFieldName"/>
+ * to disable this feature.
+ * ttlParamName - Name of an update request param this process should
+ * look for in each request when processing document additions, defaulting to
+ * _ttl_. If the the specified param name exists in an update request,
+ * the param value will be parsed as a {@linkplain DateMathParser Date Math Expression}
+ * relative to NOW and the result will be used as a default for any
+ * document included in that request that does not already have a value in the
+ * field specified by ttlFieldName. Use
+ * <null name="ttlParamName"/> to disable this feature.
+ * autoDeletePeriodSeconds - Optional numeric value indicating how
+ * often this factory should trigger a delete to remove documents. If this option is
+ * used, and specifies a non-negative numeric value, a background thread will be
+ * created that will execute recurring deleteByQuery commands using the
+ * specified period. The delete query will remove all documents with an
+ * expirationFieldName up to NOW.
+ * autoDeleteChainName - Optional name of an
+ * updateRequestProcessorChain to use when executing automatic deletes.
+ * If not specified, or <null/>, the default
+ * updateRequestProcessorChain for this collection is used.
+ * This option is ignored unless autoDeletePeriodSeconds is configured
+ * and is non-negative.
+ *
+ *
+ *
+ * For example: The configuration below will cause any document with a field named
+ * _ttl_ to have a Date field named _expire_at_ computed
+ * for it when added -- but no automatic deletion will happen.
+ *
+ *
+ *
+ * <processor class="solr.processor.DocExpirationUpdateProcessorFactory">
+ * <str name="expirationFieldName">_expire_at_</str>
+ * </processor>
+ *
+ *
+ * Alternatively, in this configuration deletes will occur automatically against the
+ * _expire_at_ field every 5 minutes - but this processor will not
+ * automatically populate the _expire_at_ using any sort of TTL expression.
+ * Only documents that were added with an explicit _expire_at_ field value
+ * will ever be deleted.
+ *
+ *
+ *
+ * <processor class="solr.processor.DocExpirationUpdateProcessorFactory">
+ * <null name="ttlFieldName"/>
+ * <null name="ttlParamName"/>
+ * <int name="autoDeletePeriodSeconds">300</int>
+ * <str name="expirationFieldName">_expire_at_</str>
+ * </processor>
+ *
+ *
+ * This last example shows the combination of both features using a custom
+ * ttlFieldName: Documents with a my_ttl field will
+ * have an _expire_at_ field computed, and deletes will be triggered
+ * every 5 minutes to remove documents whose
+ * _expire_at_ field value is in the past.
+ *
+ *
+ *
+ * <processor class="solr.processor.DocExpirationUpdateProcessorFactory">
+ * <int name="autoDeletePeriodSeconds">300</int>
+ * <str name="ttlFieldName">my_ttl</str>
+ * <null name="ttlParamName"/>
+ * <str name="expirationFieldName">_expire_at_</str>
+ * </processor>
+ */
+public final class DocExpirationUpdateProcessorFactory
+ extends UpdateRequestProcessorFactory
+ implements SolrCoreAware {
+
+ public final static Logger log = LoggerFactory.getLogger(DocExpirationUpdateProcessorFactory.class);
+
+ private static final String DEF_TTL_KEY = "_ttl_";
+ private static final String EXP_FIELD_NAME_CONF = "expirationFieldName";
+ private static final String TTL_FIELD_NAME_CONF = "ttlFieldName";
+ private static final String TTL_PARAM_NAME_CONF = "ttlParamName";
+ private static final String DEL_CHAIN_NAME_CONF = "autoDeleteChainName";
+ private static final String DEL_PERIOD_SEC_CONF = "autoDeletePeriodSeconds";
+
+ private SolrCore core;
+ private ScheduledThreadPoolExecutor executor;
+
+ private String expireField = null;
+ private String ttlField = null;
+ private String ttlParam = null;
+
+ private String deleteChainName = null;
+ private long deletePeriodSeconds = -1L;
+
+ private SolrException confErr(final String msg) {
+ return confErr(msg, null);
+ }
+ private SolrException confErr(final String msg, SolrException root) {
+ return new SolrException(SERVER_ERROR, this.getClass().getSimpleName()+": "+msg, root);
+ }
+ private String removeArgStr(final NamedList args, final String arg, final String def,
+ final String errMsg) {
+
+ if (args.indexOf(arg,0) < 0) return def;
+
+ Object tmp = args.remove(arg);
+ if (null == tmp) return null;
+
+ if (tmp instanceof String) return tmp.toString();
+
+ throw confErr(arg + " " + errMsg);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void init(NamedList args) {
+
+ deleteChainName = removeArgStr(args, DEL_CHAIN_NAME_CONF, null,
+ "must be a or or or ");
+ if (null == expireField) {
+ throw confErr(EXP_FIELD_NAME_CONF + " must be configured");
+ }
+
+ Object tmp = args.remove(DEL_PERIOD_SEC_CONF);
+ if (null != tmp) {
+ if (! (tmp instanceof Number)) {
+ throw confErr(DEL_PERIOD_SEC_CONF + " must be an or ");
+ }
+ deletePeriodSeconds = ((Number)tmp).longValue();
+ }
+
+ super.init(args);
+ }
+
+ @Override
+ public void inform(SolrCore core) {
+ this.core = core;
+
+ if (null == core.getLatestSchema().getFieldTypeNoEx(expireField)) {
+ // TODO: check for managed schema and auto-add as a date field?
+ throw confErr(EXP_FIELD_NAME_CONF + " does not exist in schema: " + expireField);
+ }
+
+ if (0 < deletePeriodSeconds) {
+ // validate that we have a chain we can work with
+ try {
+ Object ignored = core.getUpdateProcessingChain(deleteChainName);
+ } catch (SolrException e) {
+ throw confErr(DEL_CHAIN_NAME_CONF + " does not exist: " + deleteChainName, e);
+ }
+ // schedule recuring deletion
+ initDeleteExpiredDocsScheduler(core);
+ }
+ }
+
+ private void initDeleteExpiredDocsScheduler(SolrCore core) {
+ executor = new ScheduledThreadPoolExecutor
+ (1, new DefaultSolrThreadFactory("autoExpireDocs"),
+ new RejectedExecutionHandler() {
+ public void rejectedExecution(Runnable r, ThreadPoolExecutor e) {
+ log.warn("Skipping execution of '{}' using '{}'", r, e);
+ }
+ });
+
+ core.addCloseHook(new CloseHook() {
+ public void postClose(SolrCore core) {
+ // update handler is gone, hard terminiate anything that's left.
+
+ if (executor.isTerminating()) {
+ log.info("Triggering hard shutdown of DocExpiration Executor");
+ executor.shutdownNow();
+ }
+ }
+ public void preClose(SolrCore core) {
+ log.info("Triggering Graceful shutdown of DocExpiration Executor");
+ executor.shutdown();
+ }
+ });
+
+ executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
+ executor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false);
+ // we don't want this firing right away, since the core may not be ready
+ final long initialDelay = deletePeriodSeconds;
+ // TODO: should we make initialDelay configurable
+ // TODO: should we make initialDelay some fraction of the period?
+ executor.scheduleAtFixedRate(new DeleteExpiredDocsRunnable(this),
+ deletePeriodSeconds,
+ deletePeriodSeconds,
+ TimeUnit.SECONDS);
+
+ }
+
+ @Override
+ public UpdateRequestProcessor getInstance(SolrQueryRequest req,
+ SolrQueryResponse rsp,
+ UpdateRequestProcessor next ) {
+
+ String defaultTtl = (null == ttlParam) ? null : req.getParams().get(ttlParam);
+
+ if (null == ttlField && null == defaultTtl) {
+ // nothing to do, shortcircut ourselves out of the chain.
+ return next;
+ } else {
+ return new TTLUpdateProcessor(defaultTtl, expireField, ttlField, next);
+ }
+ }
+
+ private static final class TTLUpdateProcessor extends UpdateRequestProcessor {
+
+ final String defaultTtl;
+ final String expireField;
+ final String ttlField;
+ public TTLUpdateProcessor(final String defaultTtl,
+ final String expireField,
+ final String ttlField,
+ final UpdateRequestProcessor next) {
+ super(next);
+ this.defaultTtl = defaultTtl;
+ this.expireField = expireField;
+ this.ttlField = ttlField;
+ }
+
+ @Override
+ public void processAdd(AddUpdateCommand cmd) throws IOException {
+ final SolrInputDocument doc = cmd.getSolrInputDocument();
+
+ final String math = doc.containsKey(ttlField)
+ ? doc.getFieldValue(ttlField).toString() : defaultTtl;
+
+ if (null != math) {
+ try {
+ final DateMathParser dmp = new DateMathParser();
+ // TODO: should we try to accept things like "1DAY" as well as "+1DAY" ?
+ // How?
+ // 'startsWith("+")' is a bad idea because it would cause porblems with
+ // things like "/DAY+1YEAR"
+ // Maybe catch ParseException and rety with "+" prepended?
+ doc.addField(expireField, dmp.parseMath(math));
+ } catch (ParseException pe) {
+ throw new SolrException(BAD_REQUEST, "Can't parse ttl as date math: " + math, pe);
+ }
+ }
+
+ super.processAdd(cmd);
+ }
+ }
+
+ /**
+ *
+ * Runnable that uses the the deleteChainName configured for
+ * this factory to execute a delete by query (using the configured
+ * expireField) followed by a soft commit to re-open searchers (if needed)
+ *
+ *
+ * This logic is all wrapped up in a new SolrRequestInfo context with
+ * some logging to help make it obvious this background activity is happening.
+ *
+ *
+ * In cloud mode, this runner only triggers deletes if
+ * {@link #iAmInChargeOfPeriodicDeletes} is true.
+ * (logging is minimal in this situation)
+ *
+ *
+ * @see #iAmInChargeOfPeriodicDeletes
+ */
+ private static final class DeleteExpiredDocsRunnable implements Runnable {
+ final DocExpirationUpdateProcessorFactory factory;
+ final SolrCore core;
+ final String deleteChainName;
+ final String expireField;
+ public DeleteExpiredDocsRunnable(final DocExpirationUpdateProcessorFactory factory) {
+ this.factory = factory;
+ this.core = factory.core;
+ this.deleteChainName = factory.deleteChainName;
+ this.expireField = factory.expireField;
+ }
+
+ public void run() {
+ // setup the request context early so the logging (including any from
+ // shouldWeDoPeriodicDelete() ) includes the core context info
+ final SolrQueryRequest req = new LocalSolrQueryRequest
+ (factory.core, Collections.emptyMap());
+ try {
+ final SolrQueryResponse rsp = new SolrQueryResponse();
+ SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
+ try {
+
+ if (! factory.iAmInChargeOfPeriodicDeletes() ) {
+ // No-Op
+ return;
+ }
+ log.info("Begining periodic deletion of expired docs");
+
+ UpdateRequestProcessorChain chain = core.getUpdateProcessingChain(deleteChainName);
+ UpdateRequestProcessor proc = chain.createProcessor(req, rsp);
+ if (null == proc) {
+ log.warn("No active processors, skipping automatic deletion " +
+ "of expired docs using chain: {}", deleteChainName);
+ return;
+ }
+ try {
+ DeleteUpdateCommand del = new DeleteUpdateCommand(req);
+ del.setQuery("{!cache=false}" + expireField + ":[* TO " +
+ DateField.formatExternal(SolrRequestInfo.getRequestInfo().getNOW())
+ + "]");
+ proc.processDelete(del);
+
+ // TODO: should this be more configurable?
+ // TODO: in particular: should hard commit be optional?
+ CommitUpdateCommand commit = new CommitUpdateCommand(req, false);
+ commit.softCommit = true;
+ commit.openSearcher = true;
+ proc.processCommit(commit);
+
+ } finally {
+ proc.finish();
+ }
+
+ log.info("Finished periodic deletion of expired docs");
+ } catch (IOException ioe) {
+ log.error("IOException in periodic deletion of expired docs: " +
+ ioe.getMessage(), ioe);
+ // DO NOT RETHROW: ScheduledExecutor will supress subsequent executions
+ } catch (RuntimeException re) {
+ log.error("Runtime error in periodic deletion of expired docs: " +
+ re.getMessage(), re);
+ // DO NOT RETHROW: ScheduledExecutor will supress subsequent executions
+ } finally {
+ SolrRequestInfo.clearRequestInfo();
+ }
+ } finally {
+ req.close();
+ }
+ }
+ }
+
+ /**
+ *
+ * Helper method that returns true if the Runnable managed by this factory
+ * should be responseible of doing periodica deletes.
+ *
+ *
+ * In simple standalone instalations this method always returns true,
+ * but in cloud mode it will be true if and only if we are currently the leader
+ * of the (active) slice with the first name (lexigraphically).
+ *
+ *
+ * If this method returns false, it may have also logged a message letting the user
+ * know why we aren't attempting period deletion (but it will attempt to not log
+ * this excessively)
+ *
+ */
+ private boolean iAmInChargeOfPeriodicDeletes() {
+ ZkController zk = core.getCoreDescriptor().getCoreContainer().getZkController();
+
+ if (null == zk) return true;
+
+ // This is a lot simpler then doing our own "leader" election across all replicas
+ // of all shards since:
+ // a) we already have a per shard leader
+ // b) shard names must be unique
+ // c) ClusterState is already being "watched" by ZkController, no additional zk hits
+ // d) there might be multiple instances of this factory (in multiple chains) per
+ // collection, so picking an ephemeral node name for our election would be tricky
+
+ CloudDescriptor desc = core.getCoreDescriptor().getCloudDescriptor();
+ String col = desc.getCollectionName();
+
+ List slices = new ArrayList(zk.getClusterState().getActiveSlices(col));
+ Collections.sort(slices, COMPARE_SLICES_BY_NAME);
+ if (slices.isEmpty()) {
+ log.error("Collection {} has no active Slices?", col);
+ return false;
+ }
+ Replica firstSliceLeader = slices.get(0).getLeader();
+ if (null == firstSliceLeader) {
+ log.warn("Slice in charge of periodic deletes for {} does not currently have a leader",
+ col);
+ return false;
+ }
+ String leaderInCharge = firstSliceLeader.getName();
+ String myCoreNodeName = desc.getCoreNodeName();
+
+ boolean inChargeOfDeletesRightNow = leaderInCharge.equals(myCoreNodeName);
+
+ if (previouslyInChargeOfDeletes && ! inChargeOfDeletesRightNow) {
+ // don't spam the logs constantly, just log when we know that we're not the guy
+ // (the first time -- or anytime we were, but no longer are)
+ log.info("Not currently in charge of periodic deletes for this collection, " +
+ "will not trigger delete or log again until this changes");
+ }
+
+ previouslyInChargeOfDeletes = inChargeOfDeletesRightNow;
+ return inChargeOfDeletesRightNow;
+ }
+
+ /** @see #iAmInChargeOfPeriodicDeletes */
+ private volatile boolean previouslyInChargeOfDeletes = true;
+
+ private static final Comparator COMPARE_SLICES_BY_NAME = new Comparator() {
+ public int compare(Slice a, Slice b) {
+ return a.getName().compareTo(b.getName());
+ }
+ };
+
+}
+
+
+
diff --git a/solr/core/src/java/org/apache/solr/update/processor/MD5Signature.java b/solr/core/src/java/org/apache/solr/update/processor/MD5Signature.java
index be849308c7b..02a0e2b223e 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/MD5Signature.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/MD5Signature.java
@@ -17,6 +17,7 @@ package org.apache.solr.update.processor;
*/
import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
@@ -44,13 +45,7 @@ public class MD5Signature extends Signature {
@Override
public void add(String content) {
- try {
- digester.update(content.getBytes("UTF-8"));
- } catch (UnsupportedEncodingException e) {
- // won't happen
- log.error("UTF-8 not supported", e);
- throw new RuntimeException(e);
- }
+ digester.update(content.getBytes(StandardCharsets.UTF_8));
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/update/processor/RegexpBoostProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/RegexpBoostProcessor.java
index 29a7acba426..c70a2431e28 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/RegexpBoostProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/RegexpBoostProcessor.java
@@ -21,18 +21,18 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
-import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.IOUtils;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.AddUpdateCommand;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -121,7 +121,7 @@ public class RegexpBoostProcessor extends UpdateRequestProcessor {
private List initBoostEntries(InputStream is) throws IOException {
List newBoostEntries = new ArrayList<>();
- BufferedReader reader = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8")));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
try {
String line = null;
while ((line = reader.readLine()) != null) {
diff --git a/solr/core/src/java/org/apache/solr/update/processor/StatelessScriptUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/StatelessScriptUpdateProcessorFactory.java
index f330e63c34b..c634ec47ea1 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/StatelessScriptUpdateProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/StatelessScriptUpdateProcessorFactory.java
@@ -27,7 +27,6 @@ import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.*;
import org.apache.solr.util.plugin.SolrCoreAware;
-
import org.apache.commons.lang.StringUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.FilenameUtils;
@@ -41,6 +40,7 @@ import javax.script.ScriptException;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
+import java.nio.charset.StandardCharsets;
import java.util.Set;
import java.util.LinkedHashSet;
import java.util.ArrayList;
@@ -494,7 +494,7 @@ public class StatelessScriptUpdateProcessorFactory extends UpdateRequestProcesso
public Reader openReader(SolrResourceLoader resourceLoader) throws IOException {
InputStream input = resourceLoader.openResource(fileName);
return org.apache.lucene.util.IOUtils.getDecodingReader
- (input, org.apache.lucene.util.IOUtils.CHARSET_UTF_8);
+ (input, StandardCharsets.UTF_8);
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/util/SimplePostTool.java b/solr/core/src/java/org/apache/solr/util/SimplePostTool.java
index 80e54eed158..a55f6228720 100644
--- a/solr/core/src/java/org/apache/solr/util/SimplePostTool.java
+++ b/solr/core/src/java/org/apache/solr/util/SimplePostTool.java
@@ -18,37 +18,37 @@ package org.apache.solr.util;
*/
import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.ByteArrayInputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
-import java.io.UnsupportedEncodingException;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.ProtocolException;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
-import java.util.HashSet;
import java.util.TimeZone;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import java.util.zip.GZIPInputStream;
import java.util.zip.Inflater;
import java.util.zip.InflaterInputStream;
-import java.net.HttpURLConnection;
-import java.net.MalformedURLException;
-import java.net.ProtocolException;
-import java.net.URL;
-import java.net.URLEncoder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
@@ -893,13 +893,7 @@ public class SimplePostTool {
* @return the input stream
*/
public static InputStream stringToStream(String s) {
- InputStream is = null;
- try {
- is = new ByteArrayInputStream(s.getBytes("UTF-8"));
- } catch (UnsupportedEncodingException e) {
- fatal("Shouldn't happen: UTF-8 not supported?!?!?!");
- }
- return is;
+ return new ByteArrayInputStream(s.getBytes(StandardCharsets.UTF_8));
}
/**
@@ -961,10 +955,9 @@ public class SimplePostTool {
/**
* Takes a string as input and returns a DOM
*/
- public static Document makeDom(String in, String inputEncoding) throws SAXException, IOException,
+ public static Document makeDom(byte[] in) throws SAXException, IOException,
ParserConfigurationException {
- InputStream is = new ByteArrayInputStream(in
- .getBytes(inputEncoding));
+ InputStream is = new ByteArrayInputStream(in);
Document dom = DocumentBuilderFactory.newInstance()
.newDocumentBuilder().parse(is);
return dom;
@@ -1105,7 +1098,7 @@ public class SimplePostTool {
*/
protected List parseRobotsTxt(InputStream is) throws IOException {
List disallows = new ArrayList<>();
- BufferedReader r = new BufferedReader(new InputStreamReader(is, "UTF-8"));
+ BufferedReader r = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
String l;
while((l = r.readLine()) != null) {
String[] arr = l.split("#");
@@ -1137,10 +1130,9 @@ public class SimplePostTool {
URL extractUrl = new URL(appendParam(postUrl.toString(), "extractOnly=true"));
boolean success = postData(is, null, os, type, extractUrl);
if(success) {
- String rawXml = os.toString("UTF-8");
- Document d = makeDom(rawXml, "UTF-8");
+ Document d = makeDom(os.toByteArray());
String innerXml = getXP(d, "/response/str/text()[1]", false);
- d = makeDom(innerXml, "UTF-8");
+ d = makeDom(innerXml.getBytes(StandardCharsets.UTF_8));
NodeList links = getNodesFromXP(d, "/html/body//a/@href");
for(int i = 0; i < links.getLength(); i++) {
String link = links.item(i).getTextContent();
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/intblock/package.html b/solr/core/src/test-files/solr/collection1/conf/schema-preanalyzed.xml
similarity index 53%
rename from lucene/codecs/src/java/org/apache/lucene/codecs/intblock/package.html
rename to solr/core/src/test-files/solr/collection1/conf/schema-preanalyzed.xml
index 403ea1b55f6..a9422e313e6 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/intblock/package.html
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-preanalyzed.xml
@@ -1,4 +1,4 @@
-
+
-
-
-
+
+
+
+
+
+
+
+ id
+
+
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml b/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml
index 60281903716..ece1a8e9642 100755
--- a/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml
@@ -457,6 +457,7 @@
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml
index f5b711c3769..53433b0ea50 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml
@@ -71,6 +71,94 @@ NOTE: Tests expect every field in this schema to be sortable.
@@ -157,6 +307,94 @@ NOTE: Tests expect every field in this schema to be sortable.
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-doc-expire-update-processor.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-doc-expire-update-processor.xml
new file mode 100644
index 00000000000..b783a5dc62b
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-doc-expire-update-processor.xml
@@ -0,0 +1,103 @@
+
+
+
+
+
+
+
+ ${solr.data.dir:}
+
+ ${tests.luceneMatchVersion:LUCENE_CURRENT}
+
+
+
+ ${solr.ulog.dir:}
+
+
+
+
+
+ true
+
+
+
+
+ _expire_at_tdt
+
+
+
+
+
+ _ttl_field_
+ _expire_at_tdt
+
+
+ _ttl_field_
+
+
+
+
+
+ _ttl_param_
+ _expire_at_tdt
+
+
+
+
+
+ _ttl_field_
+ _ttl_param_
+ _expire_at_tdt
+
+
+ _ttl_field_
+
+
+
+
+
+
+
+ 3
+ eXpField_tdt
+ tTl_s
+
+
+
+
diff --git a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
index 0b254e5a3a2..34383c60939 100644
--- a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
+++ b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
@@ -20,6 +20,7 @@ package org.apache.solr;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -468,7 +469,7 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 {
DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
builder.parse(new ByteArrayInputStream
- (writer.toString().getBytes("UTF-8")));
+ (writer.toString().getBytes(StandardCharsets.UTF_8)));
req.close();
}
diff --git a/solr/core/src/test/org/apache/solr/CursorPagingTest.java b/solr/core/src/test/org/apache/solr/CursorPagingTest.java
index e291d8c144d..6552da7ef17 100644
--- a/solr/core/src/test/org/apache/solr/CursorPagingTest.java
+++ b/solr/core/src/test/org/apache/solr/CursorPagingTest.java
@@ -31,18 +31,22 @@ import static org.apache.solr.common.params.CursorMarkParams.CURSOR_MARK_START;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.schema.DateField;
import org.apache.solr.search.CursorMark; //jdoc
import org.noggit.ObjectBuilder;
+import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.ArrayList;
+import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Collection;
import java.util.Collections;
+import java.util.Locale;
import java.util.Map;
-import java.nio.ByteBuffer;
+import java.util.UUID;
import org.junit.BeforeClass;
import org.junit.After;
@@ -56,6 +60,9 @@ public class CursorPagingTest extends SolrTestCaseJ4 {
public final static String TEST_SOLRCONFIG_NAME = "solrconfig-deeppaging.xml";
/** schema.xml file name, shared with other cursor related tests */
public final static String TEST_SCHEMAXML_NAME = "schema-sorts.xml";
+ /** values from enumConfig.xml */
+ public static final String[] SEVERITY_ENUM_VALUES =
+ { "Not Available", "Low", "Medium", "High", "Critical" };
@BeforeClass
public static void beforeTests() throws Exception {
@@ -676,7 +683,7 @@ public class CursorPagingTest extends SolrTestCaseJ4 {
String cursorMark = CURSOR_MARK_START;
int docsOnThisPage = Integer.MAX_VALUE;
while (0 < docsOnThisPage) {
- String json = assertJQ(req(params,
+ String json = assertJQ(req(params,
CURSOR_MARK_PARAM, cursorMark));
Map rsp = (Map) ObjectBuilder.fromJSON(json);
assertTrue("response doesn't contain " + CURSOR_MARK_NEXT + ": " + json,
@@ -893,7 +900,6 @@ public class CursorPagingTest extends SolrTestCaseJ4 {
if (useField()) {
doc.addField("str", skewed(randomUsableUnicodeString(),
TestUtil.randomSimpleString(random(), 1, 1)));
-
}
if (useField()) {
int numBytes = (int) skewed(TestUtil.nextInt(random(), 20, 50), 2);
@@ -901,6 +907,23 @@ public class CursorPagingTest extends SolrTestCaseJ4 {
random().nextBytes(randBytes);
doc.addField("bin", ByteBuffer.wrap(randBytes));
}
+ if (useField()) {
+ doc.addField("date", skewed(randomDate(),
+ dateWithRandomSecondOn2010_10_31_at_10_31()));
+ }
+ if (useField()) {
+ doc.addField("uuid", UUID.randomUUID().toString());
+ }
+ if (useField()) {
+ doc.addField("currency", skewed("" + (random().nextInt() / 100.) + "," + randomCurrency(),
+ "" + TestUtil.nextInt(random(), 250, 320) + ",USD"));
+ }
+ if (useField()) {
+ doc.addField("bool", random().nextBoolean() ? "t" : "f");
+ }
+ if (useField()) {
+ doc.addField("enum", randomEnumValue());
+ }
return doc;
}
@@ -939,6 +962,25 @@ public class CursorPagingTest extends SolrTestCaseJ4 {
return result;
}
+ private static String randomDate() {
+ return DateField.formatExternal(new Date(random().nextLong()));
+ }
+
+ private static String dateWithRandomSecondOn2010_10_31_at_10_31() {
+ return String.format(Locale.ROOT, "2010-10-31T10:31:%02d.000Z",
+ TestUtil.nextInt(random(), 0, 59));
+ }
+
+ private static final String[] currencies = { "USD", "EUR", "NOK" };
+
+ public static String randomCurrency() {
+ return currencies[random().nextInt(currencies.length)];
+ }
+
+ private static String randomEnumValue() {
+ return SEVERITY_ENUM_VALUES[random().nextInt(SEVERITY_ENUM_VALUES.length)];
+ }
+
/**
* Given a list of fieldNames, builds up a random sort string which is guaranteed to
* have at least 3 clauses, ending with the "id" field for tie breaking
@@ -956,15 +998,16 @@ public class CursorPagingTest extends SolrTestCaseJ4 {
String field = shuffledNames.get(i);
// wrap in a function sometimes
- if ( (!"score".equals(field))
+ if ( (!"score".equals(field) && !field.contains("bcd"))
&&
(0 == TestUtil.nextInt(random(), 0, 7)) ) {
// specific function doesn't matter, just proving that we can handle the concept.
// but we do have to be careful with non numeric fields
- if (field.startsWith("str") || field.startsWith("bin")) {
- field = "if(exists(" + field + "),47,83)";
- } else {
+ if (field.contains("float") || field.contains("double")
+ || field.contains("int") || field.contains("long")) {
field = "abs(" + field + ")";
+ } else {
+ field = "if(exists(" + field + "),47,83)";
}
}
result.append(field).append(random().nextBoolean() ? " asc, " : " desc, ");
diff --git a/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java b/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java
index 15ee5627780..17de9263d9b 100644
--- a/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java
+++ b/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java
@@ -29,6 +29,7 @@ import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.junit.BeforeClass;
+import java.nio.charset.StandardCharsets;
/**
* Test for Loading core properties from a properties file
@@ -63,7 +64,7 @@ public class TestSolrCoreProperties extends SolrJettyTestBase {
Properties p = new Properties();
p.setProperty("foo.foo1", "f1");
p.setProperty("foo.foo2", "f2");
- Writer fos = new OutputStreamWriter(new FileOutputStream(new File(confDir, "solrcore.properties")), IOUtils.CHARSET_UTF_8);
+ Writer fos = new OutputStreamWriter(new FileOutputStream(new File(confDir, "solrcore.properties")), StandardCharsets.UTF_8);
p.store(fos, null);
IOUtils.close(fos);
diff --git a/solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java b/solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java
index 10cf2472967..58158d9a15c 100644
--- a/solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java
+++ b/solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java
@@ -23,6 +23,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
import java.util.HashSet;
import java.util.Set;
@@ -62,7 +63,7 @@ public class LegacyHTMLStripCharFilterTest extends BaseTokenStreamTestCase {
//Some sanity checks, but not a full-fledged check
public void testHTML() throws Exception {
InputStream stream = getClass().getResourceAsStream("htmlStripReaderTest.html");
- LegacyHTMLStripCharFilter reader = new LegacyHTMLStripCharFilter(new InputStreamReader(stream, "UTF-8"));
+ LegacyHTMLStripCharFilter reader = new LegacyHTMLStripCharFilter(new InputStreamReader(stream, StandardCharsets.UTF_8));
StringBuilder builder = new StringBuilder();
int ch = -1;
while ((ch = reader.read()) != -1){
diff --git a/solr/core/src/test/org/apache/solr/analytics/AbstractAnalyticsStatsTest.java b/solr/core/src/test/org/apache/solr/analytics/AbstractAnalyticsStatsTest.java
index 7be2c339823..e59758d08bb 100644
--- a/solr/core/src/test/org/apache/solr/analytics/AbstractAnalyticsStatsTest.java
+++ b/solr/core/src/test/org/apache/solr/analytics/AbstractAnalyticsStatsTest.java
@@ -21,6 +21,7 @@ import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -82,7 +83,7 @@ public class AbstractAnalyticsStatsTest extends SolrTestCaseJ4 {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setNamespaceAware(true); // never forget this!
DocumentBuilder builder = factory.newDocumentBuilder();
- doc = builder.parse(new InputSource(new ByteArrayInputStream(response.getBytes("UTF-8"))));
+ doc = builder.parse(new InputSource(new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8))));
xPathFact = XPathFactory.newInstance();
rawResponse = response;
}
@@ -106,8 +107,8 @@ public class AbstractAnalyticsStatsTest extends SolrTestCaseJ4 {
case DOUBLE: return Double.parseDouble(val);
case FLOAT: return Float.parseFloat(val);
case LONG: return Long.parseLong(val);
- case STRING: return val;
- case DATE: return val;
+ case STRING: assertTrue(rawResponse, val != null && val.length() > 0 ); return val;
+ case DATE: assertTrue(rawResponse, val != null && val.length() > 0 ); return val;
}
} catch (Exception e) {
e.printStackTrace();
diff --git a/solr/core/src/test/org/apache/solr/analytics/expression/ExpressionTest.java b/solr/core/src/test/org/apache/solr/analytics/expression/ExpressionTest.java
index 43c5f501c81..5819254b429 100644
--- a/solr/core/src/test/org/apache/solr/analytics/expression/ExpressionTest.java
+++ b/solr/core/src/test/org/apache/solr/analytics/expression/ExpressionTest.java
@@ -20,7 +20,6 @@ package org.apache.solr.analytics.expression;
import com.google.common.collect.ObjectArrays;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.LuceneTestCase.BadApple;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.analytics.AbstractAnalyticsStatsTest;
@@ -36,7 +35,6 @@ import java.util.ArrayList;
import java.util.Scanner;
@SuppressCodecs({"Lucene3x", "Lucene40", "Lucene41", "Lucene42", "Appending", "Asserting"})
-@BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5302")
public class ExpressionTest extends AbstractAnalyticsStatsTest {
private static final String fileName = "/analytics/requestFiles/expressions.txt";
diff --git a/solr/core/src/test/org/apache/solr/analytics/facet/AbstractAnalyticsFacetTest.java b/solr/core/src/test/org/apache/solr/analytics/facet/AbstractAnalyticsFacetTest.java
index 820a1c55872..7abd79a3ad9 100644
--- a/solr/core/src/test/org/apache/solr/analytics/facet/AbstractAnalyticsFacetTest.java
+++ b/solr/core/src/test/org/apache/solr/analytics/facet/AbstractAnalyticsFacetTest.java
@@ -21,6 +21,7 @@ import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -65,7 +66,7 @@ public class AbstractAnalyticsFacetTest extends SolrTestCaseJ4 {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setNamespaceAware(true); // never forget this!
DocumentBuilder builder = factory.newDocumentBuilder();
- doc = builder.parse(new InputSource(new ByteArrayInputStream(response.getBytes("UTF-8"))));
+ doc = builder.parse(new InputSource(new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8))));
xPathFact = XPathFactory.newInstance();
rawResponse = response;
}
diff --git a/solr/core/src/test/org/apache/solr/analytics/facet/FieldFacetExtrasTest.java b/solr/core/src/test/org/apache/solr/analytics/facet/FieldFacetExtrasTest.java
index 2e0b6206fba..8377ccd4822 100644
--- a/solr/core/src/test/org/apache/solr/analytics/facet/FieldFacetExtrasTest.java
+++ b/solr/core/src/test/org/apache/solr/analytics/facet/FieldFacetExtrasTest.java
@@ -24,6 +24,7 @@ import java.util.Collections;
import java.util.List;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
diff --git a/solr/core/src/test/org/apache/solr/analytics/facet/FieldFacetTest.java b/solr/core/src/test/org/apache/solr/analytics/facet/FieldFacetTest.java
index 0c28fa5af4e..12cfe37cb6a 100644
--- a/solr/core/src/test/org/apache/solr/analytics/facet/FieldFacetTest.java
+++ b/solr/core/src/test/org/apache/solr/analytics/facet/FieldFacetTest.java
@@ -18,17 +18,18 @@
package org.apache.solr.analytics.facet;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.junit.Ignore;
@SuppressCodecs({"Lucene3x","Lucene40","Lucene41","Lucene42","Appending","Asserting"})
-@Ignore // failing after https://issues.apache.org/jira/browse/SOLR-5685
public class FieldFacetTest extends AbstractAnalyticsFacetTest{
static String fileName = "/analytics/requestFiles/fieldFacets.txt";
@@ -390,8 +391,13 @@ public class FieldFacetTest extends AbstractAnalyticsFacetTest{
}
}
- assertU(commit());
- setResponse(h.query(request(fileToStringArr(FieldFacetTest.class, fileName))));
+ assertU(commit());
+ String[] reqFacetParamas = fileToStringArr(FieldFacetTest.class, fileName);
+ String[] reqParamas = new String[reqFacetParamas.length + 2];
+ System.arraycopy(reqFacetParamas, 0, reqParamas, 0, reqFacetParamas.length);
+ reqParamas[reqFacetParamas.length] = "solr";
+ reqParamas[reqFacetParamas.length+1] = "asc";
+ setResponse(h.query(request(reqFacetParamas)));
}
@SuppressWarnings("unchecked")
@@ -1063,11 +1069,18 @@ public class FieldFacetTest extends AbstractAnalyticsFacetTest{
}
private void checkStddevs(ArrayList list1, ArrayList list2) {
+ Collections.sort(list1);
+ Collections.sort(list2);
for (int i = 0; i) actual);
+ Collections.sort((List) expected);
+ Assert.assertEquals(mes, actual, expected);
+ }
}
diff --git a/solr/core/src/test/org/apache/solr/analytics/util/valuesource/FunctionTest.java b/solr/core/src/test/org/apache/solr/analytics/util/valuesource/FunctionTest.java
index bf212a118b7..29b97d3f1a8 100644
--- a/solr/core/src/test/org/apache/solr/analytics/util/valuesource/FunctionTest.java
+++ b/solr/core/src/test/org/apache/solr/analytics/util/valuesource/FunctionTest.java
@@ -90,6 +90,7 @@ public class FunctionTest extends AbstractAnalyticsStatsTest {
double result = (Double)getStatResult("ar", "sum", VAL_TYPE.DOUBLE);
double calculated = (Double)getStatResult("ar", "sumc", VAL_TYPE.DOUBLE);
assertEquals(getRawResponse(), result, calculated, 0.0);
+ // TODO checfk why asserted 2times
assertEquals(getRawResponse(), result, calculated, 0.0);
result = (Double)getStatResult("ar", "mean", VAL_TYPE.DOUBLE);
@@ -170,24 +171,24 @@ public class FunctionTest extends AbstractAnalyticsStatsTest {
@Test
public void dateMathTest() throws Exception {
- String result = (String)getStatResult("dmr", "median", VAL_TYPE.STRING);
- String calculated = (String)getStatResult("dmr", "medianc", VAL_TYPE.STRING);
+ String result = (String)getStatResult("dmr", "median", VAL_TYPE.DATE);
+ String calculated = (String)getStatResult("dmr", "medianc", VAL_TYPE.DATE);
assertEquals(getRawResponse(), result, calculated);
- result = (String)getStatResult("dmr", "max", VAL_TYPE.STRING);
- calculated = (String)getStatResult("dmr", "maxc", VAL_TYPE.STRING);
+ result = (String)getStatResult("dmr", "max", VAL_TYPE.DATE);
+ calculated = (String)getStatResult("dmr", "maxc", VAL_TYPE.DATE);
assertEquals(getRawResponse(), result, calculated);
}
@Test
public void constantDateTest() throws Exception {
- String result = (String)getStatResult("cdr", "median", VAL_TYPE.STRING);
- String calculated = (String)getStatResult("cdr", "medianc", VAL_TYPE.STRING);
+ String result = (String)getStatResult("cdr", "median", VAL_TYPE.DATE);
+ String calculated = (String)getStatResult("cdr", "medianc", VAL_TYPE.DATE);
assertEquals(getRawResponse(), result, calculated);
assertEquals(getRawResponse(), result, calculated);
- result = (String)getStatResult("cdr", "max", VAL_TYPE.STRING);
- calculated = (String)getStatResult("cdr", "maxc", VAL_TYPE.STRING);
+ result = (String)getStatResult("cdr", "max", VAL_TYPE.DATE);
+ calculated = (String)getStatResult("cdr", "maxc", VAL_TYPE.DATE);
assertEquals(getRawResponse(), result, calculated);
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/AsyncMigrateRouteKeyTest.java b/solr/core/src/test/org/apache/solr/cloud/AsyncMigrateRouteKeyTest.java
index 959d97e4c1a..6b9a7090397 100644
--- a/solr/core/src/test/org/apache/solr/cloud/AsyncMigrateRouteKeyTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/AsyncMigrateRouteKeyTest.java
@@ -53,7 +53,11 @@ public class AsyncMigrateRouteKeyTest extends MigrateRouteKeyTest {
params = new ModifiableSolrParams();
params.set("action", CollectionParams.CollectionAction.REQUESTSTATUS.toString());
params.set(OverseerCollectionProcessor.REQUESTID, asyncId);
- message = sendStatusRequestWithRetry(params, 10);
+ // This task takes long enough to run. Also check for the current state of the task to be running.
+ message = sendStatusRequestWithRetry(params, 2);
+ assertEquals("found " + asyncId + " in submitted tasks", message);
+ // Now wait until the task actually completes successfully/fails.
+ message = sendStatusRequestWithRetry(params, 20);
assertEquals("Task " + asyncId + " not found in completed tasks.",
"found " + asyncId + " in completed tasks", message);
}
@@ -92,7 +96,6 @@ public class AsyncMigrateRouteKeyTest extends MigrateRouteKeyTest {
if (state.equals("completed") || state.equals("failed"))
return (String) status.get("msg");
-
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
index 9803470e25b..9f2d84ca7a8 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java
@@ -75,14 +75,14 @@ public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase
SolrCmdDistributor.testing_errorHook = null;
}
- public static String[] fieldNames = new String[]{"f_i", "f_f", "f_d", "f_l", "f_dt"};
- public static RandVal[] randVals = new RandVal[]{rint, rfloat, rdouble, rlong, rdate};
+ protected static final String[] fieldNames = new String[]{"f_i", "f_f", "f_d", "f_l", "f_dt"};
+ protected static final RandVal[] randVals = new RandVal[]{rint, rfloat, rdouble, rlong, rdate};
- protected String[] getFieldNames() {
+ public String[] getFieldNames() {
return fieldNames;
}
- protected RandVal[] getRandValues() {
+ public RandVal[] getRandValues() {
return randVals;
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
index 19ae3ed13a7..40137955ad3 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
@@ -58,14 +58,14 @@ public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase {
SolrCmdDistributor.testing_errorHook = null;
}
- public static String[] fieldNames = new String[]{"f_i", "f_f", "f_d", "f_l", "f_dt"};
- public static RandVal[] randVals = new RandVal[]{rint, rfloat, rdouble, rlong, rdate};
+ protected static final String[] fieldNames = new String[]{"f_i", "f_f", "f_d", "f_l", "f_dt"};
+ protected static final RandVal[] randVals = new RandVal[]{rint, rfloat, rdouble, rlong, rdate};
- protected String[] getFieldNames() {
+ public String[] getFieldNames() {
return fieldNames;
}
- protected RandVal[] getRandValues() {
+ public RandVal[] getRandValues() {
return randVals;
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java
index 8a044a5afd5..6885db951b7 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java
@@ -256,7 +256,7 @@ public class ChaosMonkeyShardSplitTest extends ShardSplitTest {
// TODO: close Overseer
Overseer overseer = new Overseer(
- new HttpShardHandlerFactory().getShardHandler(), "/admin/cores", reader);
+ new HttpShardHandlerFactory().getShardHandler(), "/admin/cores", reader,null);
overseer.close();
ElectionContext ec = new OverseerElectionContext(zkClient, overseer,
address.replaceAll("/", "_"));
diff --git a/solr/core/src/test/org/apache/solr/cloud/DistribDocExpirationUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/cloud/DistribDocExpirationUpdateProcessorTest.java
new file mode 100644
index 00000000000..d532d358a61
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/DistribDocExpirationUpdateProcessorTest.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cloud;
+
+import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.TestUtil;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+
+import org.apache.solr.update.processor.DocExpirationUpdateProcessorFactory; // jdoc
+import org.apache.solr.update.processor.DocExpirationUpdateProcessorFactoryTest;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.HashMap;
+
+/** Test of {@link DocExpirationUpdateProcessorFactory} in a cloud setup */
+@Slow // Has to do some sleeping to wait for a future expiration
+public class DistribDocExpirationUpdateProcessorTest extends AbstractFullDistribZkTestBase {
+
+ public static Logger log = LoggerFactory.getLogger(DistribDocExpirationUpdateProcessorTest.class);
+
+ public DistribDocExpirationUpdateProcessorTest() {
+ configString = DocExpirationUpdateProcessorFactoryTest.CONFIG_XML;
+ schemaString = DocExpirationUpdateProcessorFactoryTest.SCHEMA_XML;
+ }
+
+ @Override
+ protected String getCloudSolrConfig() {
+ return configString;
+ }
+
+ @Override
+ public void doTest() throws Exception {
+ assertTrue("only one shard?!?!?!", 1 < shardToJetty.keySet().size());
+ log.info("number of shards: {}", shardToJetty.keySet().size());
+
+ handle.clear();
+ handle.put("maxScore", SKIPVAL);
+ handle.put("QTime", SKIPVAL);
+ handle.put("timestamp", SKIPVAL);
+
+ // some docs with no expiration
+ for (int i = 1; i <= 100; i++) {
+ indexDoc(sdoc("id", i));
+ }
+ commit();
+ waitForThingsToLevelOut(30);
+
+ // this doc better not already exist
+ waitForNoResults(0, params("q","id:999","rows","0","_trace","sanity_check"));
+
+ // record the indexversion for each server so we can check later
+ // that it only changes for one shard
+ final Map initIndexVersions = getIndexVersionOfAllReplicas();
+ assertTrue("WTF? no versions?", 0 < initIndexVersions.size());
+
+
+ // add a doc with a short TTL
+ indexDoc(sdoc("id", "999", "tTl_s","+30SECONDS"));
+ commit();
+
+ // wait for one doc to be deleted
+ waitForNoResults(180, params("q","id:999","rows","0","_trace","did_it_expire_yet"));
+
+ // verify only one shard changed
+ waitForThingsToLevelOut(30);
+ final Map finalIndexVersions = getIndexVersionOfAllReplicas();
+ assertEquals("WTF? not same num versions?",
+ initIndexVersions.size(),
+ finalIndexVersions.size());
+
+ final Set nodesThatChange = new HashSet();
+ final Set shardsThatChange = new HashSet();
+
+ int coresCompared = 0;
+ for (String shard : shardToJetty.keySet()) {
+ for (CloudJettyRunner replicaRunner : shardToJetty.get(shard)) {
+ coresCompared++;
+
+ String core = replicaRunner.coreNodeName;
+ Long initVersion = initIndexVersions.get(core);
+ Long finalVersion = finalIndexVersions.get(core);
+ assertNotNull(shard + ": no init version for core: " + core, initVersion);
+ assertNotNull(shard + ": no final version for core: " + core, finalVersion);
+
+ if (!initVersion.equals(finalVersion)) {
+ nodesThatChange.add(core + "("+shard+")");
+ shardsThatChange.add(shard);
+ }
+ }
+ }
+
+ assertEquals("Exactly one shard should have changed, instead: " + shardsThatChange
+ + " nodes=(" + nodesThatChange + ")",
+ 1, shardsThatChange.size());
+ assertEquals("somehow we missed some cores?",
+ initIndexVersions.size(), coresCompared);
+
+ // TODO: above logic verifies that deleteByQuery happens on all nodes, and ...
+ // doesn't affect searcher re-open on shards w/o expired docs ... can we also verify
+ // that *only* one node is sending the deletes ?
+ // (ie: no flood of redundent deletes?)
+
+ }
+
+ /**
+ * returns a map whose key is the coreNodeName and whose value is what the replication
+ * handler returns for the indexversion
+ */
+ private Map getIndexVersionOfAllReplicas() throws IOException, SolrServerException {
+ Map results = new HashMap();
+
+ for (List listOfReplicas : shardToJetty.values()) {
+ for (CloudJettyRunner replicaRunner : listOfReplicas) {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("command","indexversion");
+ params.set("_trace","getIndexVersion");
+ params.set("qt","/replication");
+ QueryRequest req = new QueryRequest(params);
+
+ NamedList res = replicaRunner.client.solrClient.request(req);
+ assertNotNull("null response from server: " + replicaRunner.coreNodeName, res);
+
+ Object version = res.get("indexversion");
+ assertNotNull("null version from server: " + replicaRunner.coreNodeName, version);
+ assertTrue("version isn't a long: "+replicaRunner.coreNodeName,
+ version instanceof Long);
+ results.put(replicaRunner.coreNodeName, (Long)version);
+
+ long numDocs = replicaRunner.client.solrClient.query
+ (params("q","*:*","distrib","false","rows","0","_trace","counting_docs"))
+ .getResults().getNumFound();
+ log.info("core=" + replicaRunner.coreNodeName + "; ver=" + version +
+ "; numDocs=" + numDocs);
+
+ }
+ }
+
+ return results;
+ }
+
+ /**
+ * Executes a query over and over against the cloudClient every 5 seconds
+ * until the numFound is 0 or the maxTimeLimitSeconds is exceeded.
+ * Query is garunteed to be executed at least once.
+ */
+ private void waitForNoResults(int maxTimeLimitSeconds,
+ SolrParams params)
+ throws SolrServerException, InterruptedException {
+
+ final long giveUpAfter = System.currentTimeMillis() + (1000L * maxTimeLimitSeconds);
+ long numFound = cloudClient.query(params).getResults().getNumFound();
+ while (0L < numFound && System.currentTimeMillis() < giveUpAfter) {
+ Thread.sleep(Math.min(5000, giveUpAfter - System.currentTimeMillis()));
+ numFound = cloudClient.query(params).getResults().getNumFound();
+ }
+ assertEquals("Give up waiting for no results: " + params,
+ 0L, numFound);
+ }
+
+}
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
index 777a17886d0..14a9d254d8d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
@@ -39,12 +39,17 @@ import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrServer;
import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.zookeeper.data.Stat;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
+
@LuceneTestCase.Slow
@SuppressSSL // Currently unknown why SSL does not work
public class OverseerRolesTest extends AbstractFullDistribZkTestBase{
@@ -85,11 +90,43 @@ public class OverseerRolesTest extends AbstractFullDistribZkTestBase{
@Override
public void doTest() throws Exception {
- addOverseerRole2ExistingNodes();
+ testOverseerRole();
+ testQuitCommand();
}
- private void addOverseerRole2ExistingNodes() throws Exception {
+ private void testQuitCommand() throws Exception{
+ String collectionName = "testOverseerQuit";
+
+ createCollection(collectionName, client);
+
+ waitForRecoveriesToFinish(collectionName, false);
+
+ SolrZkClient zk = client.getZkStateReader().getZkClient();
+ byte[] data = new byte[0];
+ data = zk.getData("/overseer_elect/leader", null, new Stat(), true);
+ Map m = (Map) ZkStateReader.fromJSON(data);
+ String s = (String) m.get("id");
+ String leader = LeaderElector.getNodeName(s);
+ Overseer.getInQueue(zk).offer(ZkStateReader.toJSON(new ZkNodeProps(Overseer.QUEUE_OPERATION, Overseer.QUIT)));
+ long timeout = System.currentTimeMillis()+5000;
+ String newLeader=null;
+ for(;System.currentTimeMillis() < timeout;){
+ newLeader = OverseerCollectionProcessor.getLeaderNode(zk);
+ if(!newLeader.equals(leader)) break;
+ Thread.sleep(100);
+ }
+ assertNotSame( "Leader not changed yet",newLeader,leader);
+
+
+
+ assertTrue("The old leader should have rejoined election ", OverseerCollectionProcessor.getSortedOverseerNodeNames(zk).contains(leader));
+ }
+
+
+
+
+ private void testOverseerRole() throws Exception {
String collectionName = "testOverseerCol";
createCollection(collectionName, client);
@@ -202,13 +239,6 @@ public class OverseerRolesTest extends AbstractFullDistribZkTestBase{
assertTrue("New overseer not the frontrunner : "+ getSortedOverseerNodeNames(client.getZkStateReader().getZkClient()) + " expected : "+ killedOverseer, leaderchanged);
-
-
-
-
- client.shutdown();
-
-
}
private void setOverseerRole(CollectionAction action, String overseerDesignate) throws Exception, IOException {
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
index 3ce5aac09e0..8836a7cc3a4 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
@@ -983,7 +983,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
overseers.get(overseers.size() -1).getZkStateReader().getZkClient().close();
}
Overseer overseer = new Overseer(
- new HttpShardHandlerFactory().getShardHandler(), "/admin/cores", reader);
+ new HttpShardHandlerFactory().getShardHandler(), "/admin/cores", reader,null);
overseers.add(overseer);
ElectionContext ec = new OverseerElectionContext(zkClient, overseer,
address.replaceAll("/", "_"));
diff --git a/solr/core/src/test/org/apache/solr/cloud/SolrXmlInZkTest.java b/solr/core/src/test/org/apache/solr/cloud/SolrXmlInZkTest.java
index 397fe9fd373..45f05b0eb4b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SolrXmlInZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SolrXmlInZkTest.java
@@ -19,6 +19,7 @@ package org.apache.solr.cloud;
import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.io.FileUtils;
import org.apache.solr.SolrTestCaseJ4;
@@ -35,7 +36,6 @@ import org.junit.rules.RuleChain;
import org.junit.rules.TestRule;
import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;
-import com.google.common.base.Charsets;
public class SolrXmlInZkTest extends SolrTestCaseJ4 {
@@ -84,7 +84,7 @@ public class SolrXmlInZkTest extends SolrTestCaseJ4 {
zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
if (toZk) {
- zkClient.makePath("solr.xml", XML_FOR_ZK.getBytes(Charsets.UTF_8), true);
+ zkClient.makePath("solr.xml", XML_FOR_ZK.getBytes(StandardCharsets.UTF_8), true);
}
zkClient.close();
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
new file mode 100644
index 00000000000..329ec34370f
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
@@ -0,0 +1,241 @@
+package org.apache.solr.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import com.google.common.collect.Lists;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrServer;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.common.util.NamedList;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class TestCollectionAPI extends AbstractFullDistribZkTestBase {
+
+ public static final String COLLECTION_NAME = "testcollection";
+ public static final String COLLECTION_NAME1 = "testcollection1";
+
+ public TestCollectionAPI() {
+ schemaString = "schema15.xml"; // we need a string id
+ }
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ fixShardCount = true;
+ sliceCount = 2;
+ shardCount = 2;
+ super.setUp();
+ }
+
+ @Override
+ public void doTest() throws Exception {
+ CloudSolrServer client = createCloudClient(null);
+ try {
+ createCollection(null, COLLECTION_NAME, 2, 1, 1, client, null, "conf1");
+ createCollection(null, COLLECTION_NAME1, 1, 1, 1, client, null, "conf1");
+ } finally {
+ //remove collections
+ client.shutdown();
+ }
+
+ listCollection();
+ clusterStatusNoCollection();
+ clusterStatusWithCollection();
+ clusterStatusWithCollectionAndShard();
+ clusterStatusWithRouteKey();
+ clusterStatusAliasTest();
+ }
+
+ private void clusterStatusWithCollectionAndShard() throws IOException, SolrServerException {
+ CloudSolrServer client = createCloudClient(null);
+ try {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionParams.CollectionAction.CLUSTERSTATUS.toString());
+ params.set("collection", COLLECTION_NAME);
+ params.set("shard", SHARD1);
+ SolrRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+
+ NamedList rsp = client.request(request);
+ NamedList cluster = (NamedList) rsp.get("cluster");
+ assertNotNull("Cluster state should not be null", cluster);
+ NamedList collections = (NamedList) cluster.get("collections");
+ assertNotNull("Collections should not be null in cluster state", collections);
+ assertNotNull(collections.get(COLLECTION_NAME));
+ assertEquals(1, collections.size());
+ Map collection = (Map) collections.get(COLLECTION_NAME);
+ Map shardStatus = (Map) collection.get("shards");
+ assertEquals(1, shardStatus.size());
+ Map selectedShardStatus = (Map) shardStatus.get(SHARD1);
+ assertNotNull(selectedShardStatus);
+
+ } finally {
+ //remove collections
+ client.shutdown();
+ }
+ }
+
+
+ private void listCollection() throws IOException, SolrServerException {
+ CloudSolrServer client = createCloudClient(null);
+ try {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionParams.CollectionAction.LIST.toString());
+ SolrRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+
+ NamedList rsp = client.request(request);
+ List collections = (List) rsp.get("collections");
+ assertTrue("control_collection was not found in list", collections.contains("control_collection"));
+ assertTrue(DEFAULT_COLLECTION + " was not found in list", collections.contains(DEFAULT_COLLECTION));
+ assertTrue(COLLECTION_NAME + " was not found in list", collections.contains(COLLECTION_NAME));
+ assertTrue(COLLECTION_NAME1 + " was not found in list", collections.contains(COLLECTION_NAME1));
+ } finally {
+ //remove collections
+ client.shutdown();
+ }
+
+
+ }
+
+ private void clusterStatusNoCollection() throws Exception {
+ CloudSolrServer client = createCloudClient(null);
+ try {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionParams.CollectionAction.CLUSTERSTATUS.toString());
+ SolrRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+
+ NamedList rsp = client.request(request);
+ NamedList cluster = (NamedList) rsp.get("cluster");
+ assertNotNull("Cluster state should not be null", cluster);
+ NamedList collections = (NamedList) cluster.get("collections");
+ assertNotNull("Collections should not be null in cluster state", collections);
+ assertNotNull(collections.get(COLLECTION_NAME1));
+ assertEquals(4, collections.size());
+
+ } finally {
+ //remove collections
+ client.shutdown();
+ }
+
+ }
+
+ private void clusterStatusWithCollection() throws IOException, SolrServerException {
+ CloudSolrServer client = createCloudClient(null);
+ try {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionParams.CollectionAction.CLUSTERSTATUS.toString());
+ params.set("collection", COLLECTION_NAME);
+ SolrRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+
+ NamedList rsp = client.request(request);
+ NamedList cluster = (NamedList) rsp.get("cluster");
+ assertNotNull("Cluster state should not be null", cluster);
+ NamedList collections = (NamedList) cluster.get("collections");
+ assertNotNull("Collections should not be null in cluster state", collections);
+ assertNotNull(collections.get(COLLECTION_NAME));
+ assertEquals(1, collections.size());
+ } finally {
+ //remove collections
+ client.shutdown();
+ }
+ }
+
+ private void clusterStatusWithRouteKey() throws IOException, SolrServerException {
+ CloudSolrServer client = createCloudClient(DEFAULT_COLLECTION);
+ try {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", "a!123"); // goes to shard2. see ShardRoutingTest for details
+ client.add(doc);
+ client.commit();
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionParams.CollectionAction.CLUSTERSTATUS.toString());
+ params.set("collection", DEFAULT_COLLECTION);
+ params.set(ShardParams._ROUTE_, "a!");
+ SolrRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+
+ NamedList rsp = client.request(request);
+ NamedList cluster = (NamedList) rsp.get("cluster");
+ assertNotNull("Cluster state should not be null", cluster);
+ NamedList collections = (NamedList) cluster.get("collections");
+ assertNotNull("Collections should not be null in cluster state", collections);
+ assertNotNull(collections.get(DEFAULT_COLLECTION));
+ assertEquals(1, collections.size());
+ Map collection = (Map) collections.get(DEFAULT_COLLECTION);
+ Map shardStatus = (Map) collection.get("shards");
+ assertEquals(1, shardStatus.size());
+ Map selectedShardStatus = (Map) shardStatus.get(SHARD2);
+ assertNotNull(selectedShardStatus);
+ } finally {
+ //remove collections
+ client.shutdown();
+ }
+ }
+
+ private void clusterStatusAliasTest() throws Exception {
+ CloudSolrServer client = createCloudClient(null);
+ try {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionParams.CollectionAction.CREATEALIAS.toString());
+ params.set("name", "myalias");
+ params.set("collections", DEFAULT_COLLECTION + "," + COLLECTION_NAME);
+ SolrRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+ client.request(request);
+ params = new ModifiableSolrParams();
+ params.set("action", CollectionParams.CollectionAction.CLUSTERSTATUS.toString());
+ params.set("collection", DEFAULT_COLLECTION);
+ request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+
+ NamedList rsp = client.request(request);
+
+
+ NamedList cluster = (NamedList) rsp.get("cluster");
+ assertNotNull("Cluster state should not be null", cluster);
+ Map aliases = (Map) cluster.get("aliases");
+ assertNotNull("Aliases should not be null", aliases);
+ assertEquals("Alias: myalias not found in cluster status",
+ DEFAULT_COLLECTION + "," + COLLECTION_NAME, aliases.get("myalias"));
+
+ NamedList collections = (NamedList) cluster.get("collections");
+ assertNotNull("Collections should not be null in cluster state", collections);
+ assertNotNull(collections.get(DEFAULT_COLLECTION));
+ Map collection = (Map) collections.get(DEFAULT_COLLECTION);
+ List collAlias = (List) collection.get("aliases");
+ assertEquals("Aliases not found", Lists.newArrayList("myalias"), collAlias);
+ } finally {
+ //remove collections
+ client.shutdown();
+ }
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestModifyConfFiles.java b/solr/core/src/test/org/apache/solr/cloud/TestModifyConfFiles.java
index d92e3e87c04..c572a7edf51 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestModifyConfFiles.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestModifyConfFiles.java
@@ -26,6 +26,7 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import java.io.File;
+import java.nio.charset.StandardCharsets;
public class TestModifyConfFiles extends AbstractFullDistribZkTestBase {
@@ -93,7 +94,7 @@ public class TestModifyConfFiles extends AbstractFullDistribZkTestBase {
client.request(request);
SolrZkClient zkClient = cloudClient.getZkStateReader().getZkClient();
- String contents = new String(zkClient.getData("/configs/conf1/schema.xml", null, null, true), "UTF-8");
+ String contents = new String(zkClient.getData("/configs/conf1/schema.xml", null, null, true), StandardCharsets.UTF_8);
assertTrue("Schema contents should have changed!", contents.contains(""));
@@ -107,7 +108,7 @@ public class TestModifyConfFiles extends AbstractFullDistribZkTestBase {
client.request(request);
- contents = new String(zkClient.getData("/configs/conf1/velocity/test.vm", null, null, true), "UTF-8");
+ contents = new String(zkClient.getData("/configs/conf1/velocity/test.vm", null, null, true), StandardCharsets.UTF_8);
assertTrue("Should have found new content in a velocity/test.vm.",
contents.indexOf("Some bogus stuff for a test.") != -1);
diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
index 08bbd3e02bb..26f9894a566 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
@@ -21,6 +21,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.List;
@@ -156,7 +157,7 @@ public class ZkCLITest extends SolrTestCaseJ4 {
zkClient.getData("/data.txt", null, null, true);
- assertArrayEquals(zkClient.getData("/data.txt", null, null, true), data.getBytes("UTF-8"));
+ assertArrayEquals(zkClient.getData("/data.txt", null, null, true), data.getBytes(StandardCharsets.UTF_8));
}
@Test
@@ -166,12 +167,12 @@ public class ZkCLITest extends SolrTestCaseJ4 {
"putfile", "/solr.xml", SOLR_HOME + File.separator + "solr-stress-new.xml"};
ZkCLI.main(args);
- String fromZk = new String(zkClient.getData("/solr.xml", null, null, true), "UTF-8");
+ String fromZk = new String(zkClient.getData("/solr.xml", null, null, true), StandardCharsets.UTF_8);
File locFile = new File(SOLR_HOME + File.separator + "solr-stress-new.xml");
InputStream is = new FileInputStream(locFile);
String fromLoc;
try {
- fromLoc = new String(IOUtils.toByteArray(is), "UTF-8");
+ fromLoc = new String(IOUtils.toByteArray(is), StandardCharsets.UTF_8);
} finally {
IOUtils.closeQuietly(is);
}
@@ -268,7 +269,7 @@ public class ZkCLITest extends SolrTestCaseJ4 {
@Test
public void testGet() throws Exception {
String getNode = "/getNode";
- byte [] data = new String("getNode-data").getBytes("UTF-8");
+ byte [] data = new String("getNode-data").getBytes(StandardCharsets.UTF_8);
this.zkClient.create(getNode, data, CreateMode.PERSISTENT, true);
String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
"get", getNode};
@@ -280,7 +281,7 @@ public class ZkCLITest extends SolrTestCaseJ4 {
File tmpDir = createTempDir();
String getNode = "/getFileNode";
- byte [] data = new String("getFileNode-data").getBytes("UTF-8");
+ byte [] data = new String("getFileNode-data").getBytes(StandardCharsets.UTF_8);
this.zkClient.create(getNode, data, CreateMode.PERSISTENT, true);
File file = new File(tmpDir,
diff --git a/solr/core/src/test/org/apache/solr/core/CoreContainerCoreInitFailuresTest.java b/solr/core/src/test/org/apache/solr/core/CoreContainerCoreInitFailuresTest.java
index 1ca5f2e0895..0b1b77b5328 100644
--- a/solr/core/src/test/org/apache/solr/core/CoreContainerCoreInitFailuresTest.java
+++ b/solr/core/src/test/org/apache/solr/core/CoreContainerCoreInitFailuresTest.java
@@ -58,7 +58,7 @@ public class CoreContainerCoreInitFailuresTest extends SolrTestCaseJ4 {
// solr.xml
File solrXml = new File(solrHome, "solr.xml");
- FileUtils.write(solrXml, EMPTY_SOLR_XML, IOUtils.CHARSET_UTF_8.toString());
+ FileUtils.write(solrXml, EMPTY_SOLR_XML, IOUtils.UTF_8);
// ----
// init the CoreContainer
@@ -133,7 +133,7 @@ public class CoreContainerCoreInitFailuresTest extends SolrTestCaseJ4 {
// start with two collections: one valid, and one broken
File solrXml = new File(solrHome, "solr.xml");
- FileUtils.write(solrXml, BAD_SOLR_XML, IOUtils.CHARSET_UTF_8.toString());
+ FileUtils.write(solrXml, BAD_SOLR_XML, IOUtils.UTF_8);
// our "ok" collection
FileUtils.copyFile(getFile("solr/collection1/conf/solrconfig-defaults.xml"),
@@ -272,7 +272,7 @@ public class CoreContainerCoreInitFailuresTest extends SolrTestCaseJ4 {
FileUtils.write
(FileUtils.getFile(solrHome, "col_bad", "conf", "solrconfig.xml"),
"This is giberish, not valid XML <",
- IOUtils.CHARSET_UTF_8.toString());
+ IOUtils.UTF_8);
try {
ignoreException(Pattern.quote("SAX"));
diff --git a/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java b/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java
index 542af843db7..3d3f00bb5dd 100644
--- a/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java
+++ b/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java
@@ -21,6 +21,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import java.util.Properties;
import javax.xml.parsers.ParserConfigurationException;
@@ -101,7 +102,7 @@ public class TestArbitraryIndexDir extends AbstractSolrTestCase{
p.put("index", newDir.getName());
Writer os = null;
try {
- os = new OutputStreamWriter(new FileOutputStream(idxprops), IOUtils.CHARSET_UTF_8);
+ os = new OutputStreamWriter(new FileOutputStream(idxprops), StandardCharsets.UTF_8);
p.store(os, "index properties");
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
diff --git a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java
index 65f872fd2cc..3f83e61cf5b 100644
--- a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java
+++ b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java
@@ -25,6 +25,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.jar.JarEntry;
@@ -33,7 +34,6 @@ import java.util.jar.JarOutputStream;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.io.FileUtils;
-import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.handler.admin.CollectionsHandler;
@@ -44,6 +44,7 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.xml.sax.SAXException;
+
public class TestCoreContainer extends SolrTestCaseJ4 {
private static String oldSolrHome;
@@ -192,15 +193,22 @@ public class TestCoreContainer extends SolrTestCaseJ4 {
assertNotNull(h.getCoreContainer().getLogging());
}
- private void SetUpHome(File solrHomeDirectory, String xmlFile)
- throws IOException {
- File solrXmlFile = new File(solrHomeDirectory, "solr.xml");
- BufferedWriter out = new BufferedWriter(new OutputStreamWriter(
- new FileOutputStream(solrXmlFile), IOUtils.CHARSET_UTF_8));
- out.write(xmlFile);
- out.close();
-
- // init
+ private void SetUpHome(File solrHomeDirectory, String xmlFile) throws IOException {
+ if (solrHomeDirectory.exists()) {
+ FileUtils.deleteDirectory(solrHomeDirectory);
+ }
+ assertTrue("Failed to mkdirs workDir", solrHomeDirectory.mkdirs());
+ try {
+ File solrXmlFile = new File(solrHomeDirectory, "solr.xml");
+ BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(solrXmlFile), StandardCharsets.UTF_8));
+ out.write(xmlFile);
+ out.close();
+ } catch (IOException e) {
+ FileUtils.deleteDirectory(solrHomeDirectory);
+ throw e;
+ }
+
+ //init
System.setProperty(SOLR_HOME_PROP, solrHomeDirectory.getAbsolutePath());
}
diff --git a/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java b/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java
index f7375fc5329..60d2456fbcc 100644
--- a/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java
+++ b/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java
@@ -31,6 +31,8 @@ import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.nio.charset.StandardCharsets;
+
public class TestCoreDiscovery extends SolrTestCaseJ4 {
@BeforeClass
@@ -47,7 +49,7 @@ public class TestCoreDiscovery extends SolrTestCaseJ4 {
xmlStr = xmlStr.replace("", " " + alternateCoreDir + " ");
}
File tmpFile = new File(solrHomeDirectory, ConfigSolr.SOLR_XML_FILE);
- FileUtils.write(tmpFile, xmlStr, IOUtils.CHARSET_UTF_8.toString());
+ FileUtils.write(tmpFile, xmlStr, IOUtils.UTF_8);
}
@@ -75,7 +77,7 @@ public class TestCoreDiscovery extends SolrTestCaseJ4 {
private void addCoreWithProps(Properties stockProps, File propFile) throws Exception {
if (!propFile.getParentFile().exists()) propFile.getParentFile().mkdirs();
- Writer out = new OutputStreamWriter(new FileOutputStream(propFile), IOUtils.CHARSET_UTF_8);
+ Writer out = new OutputStreamWriter(new FileOutputStream(propFile), StandardCharsets.UTF_8);
try {
stockProps.store(out, null);
} finally {
diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrXMLSerializer.java b/solr/core/src/test/org/apache/solr/core/TestSolrXMLSerializer.java
index 57c427399ae..056f9745b46 100644
--- a/solr/core/src/test/org/apache/solr/core/TestSolrXMLSerializer.java
+++ b/solr/core/src/test/org/apache/solr/core/TestSolrXMLSerializer.java
@@ -23,6 +23,7 @@ import java.io.File;
import java.io.IOException;
import java.io.StringWriter;
import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -72,21 +73,21 @@ public class TestSolrXMLSerializer extends SolrTestCaseJ4 {
sharedLibVal, adminPathKey, adminPathVal, shareSchemaKey,
shareSchemaVal, instanceDirKey, instanceDirVal);
- Writer w = new StringWriter();
+ StringWriter w = new StringWriter();
try {
serializer.persist(w, solrXMLDef);
} finally {
w.close();
}
- assertResults(((StringWriter) w).getBuffer().toString().getBytes("UTF-8"));
+ assertResults(w.toString().getBytes(StandardCharsets.UTF_8));
// again with default file
File tmpFile = TestUtil.createTempFile("solr.xml", null, createTempDir());
serializer.persistFile(tmpFile, solrXMLDef);
- assertResults(FileUtils.readFileToString(tmpFile, "UTF-8").getBytes("UTF-8"));
+ assertResults(FileUtils.readFileToByteArray(tmpFile));
tmpFile.delete();
}
diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrXmlPersistence.java b/solr/core/src/test/org/apache/solr/core/TestSolrXmlPersistence.java
index 683ce99cae6..f9a5dad4fc2 100644
--- a/solr/core/src/test/org/apache/solr/core/TestSolrXmlPersistence.java
+++ b/solr/core/src/test/org/apache/solr/core/TestSolrXmlPersistence.java
@@ -31,6 +31,7 @@ import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
+import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.util.IOUtils;
@@ -52,8 +53,8 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
-import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;
import com.google.common.base.Charsets;
+import java.nio.charset.StandardCharsets;
public class TestSolrXmlPersistence extends SolrTestCaseJ4 {
@@ -75,7 +76,7 @@ public class TestSolrXmlPersistence extends SolrTestCaseJ4 {
}
File solrXml = new File(solrHomeDirectory, "solr.xml");
- FileUtils.write(solrXml, solrXmlString, IOUtils.CHARSET_UTF_8.toString());
+ FileUtils.write(solrXml, solrXmlString, IOUtils.UTF_8);
final CoreContainer cores = createCoreContainer(solrHomeDirectory.getAbsolutePath(), solrXmlString);
return cores;
@@ -401,7 +402,7 @@ public class TestSolrXmlPersistence extends SolrTestCaseJ4 {
String defXml = FileUtils.readFileToString(
new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"),
- Charsets.UTF_8.toString());
+ StandardCharsets.UTF_8.name());
final CoreContainer cores = init(defXml, "collection1");
SolrXMLCoresLocator.NonPersistingLocator locator
= (SolrXMLCoresLocator.NonPersistingLocator) cores.getCoresLocator();
@@ -504,7 +505,7 @@ public class TestSolrXmlPersistence extends SolrTestCaseJ4 {
}
private String[] getAllNodes(String xmlString) throws ParserConfigurationException, IOException, SAXException {
- return getAllNodes(new ByteArrayInputStream(xmlString.getBytes(Charsets.UTF_8)));
+ return getAllNodes(new ByteArrayInputStream(xmlString.getBytes(StandardCharsets.UTF_8)));
}
/*
diff --git a/solr/core/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java
index 5d909131449..c68fb718d79 100644
--- a/solr/core/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java
@@ -37,6 +37,7 @@ import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.io.IOException;
import java.io.Reader;
+import java.nio.charset.StandardCharsets;
/**
* A test for {@link DocumentAnalysisRequestHandler}.
@@ -145,7 +146,7 @@ public class DocumentAnalysisRequestHandlerTest extends AnalysisRequestHandlerTe
" Müller \r\n" +
" " +
""
- ).getBytes("ISO-8859-1");
+ ).getBytes(StandardCharsets.ISO_8859_1);
// we declare a content stream without charset:
final ContentStream cs = new ByteStream(xmlBytes, "application/xml");
@@ -177,7 +178,7 @@ public class DocumentAnalysisRequestHandlerTest extends AnalysisRequestHandlerTe
" Müller \r\n" +
" " +
""
- ).getBytes("ISO-8859-1");
+ ).getBytes(StandardCharsets.ISO_8859_1);
// we declare a content stream with charset:
final ContentStream cs = new ByteStream(xmlBytes, "application/xml; charset=ISO-8859-1");
diff --git a/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java b/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java
index 2eb1b066a29..3f9c715147b 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java
@@ -29,6 +29,7 @@ import org.junit.BeforeClass;
import org.junit.Test;
import java.io.*;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.ArrayList;
@@ -41,7 +42,6 @@ public class TestCSVLoader extends SolrTestCaseJ4 {
}
String filename;
- String def_charset = "UTF-8";
File file;
@Override
@@ -66,12 +66,8 @@ public class TestCSVLoader extends SolrTestCaseJ4 {
}
void makeFile(String contents) {
- makeFile(contents,def_charset);
- }
-
- void makeFile(String contents, String charset) {
try {
- Writer out = new OutputStreamWriter(new FileOutputStream(filename), charset);
+ Writer out = new OutputStreamWriter(new FileOutputStream(filename), StandardCharsets.UTF_8);
out.write(contents);
out.close();
} catch (Exception e) {
diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
index 9e6f99c3413..08e20773aed 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
@@ -28,6 +28,7 @@ import java.io.OutputStreamWriter;
import java.io.Writer;
import java.net.MalformedURLException;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -1473,8 +1474,8 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
* character copy of file using UTF-8. If port is non-null, will be substituted any time "TEST_PORT" is found.
*/
private static void copyFile(File src, File dst, Integer port, boolean internalCompression) throws IOException {
- BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(src), "UTF-8"));
- Writer out = new OutputStreamWriter(new FileOutputStream(dst), "UTF-8");
+ BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(src), StandardCharsets.UTF_8));
+ Writer out = new OutputStreamWriter(new FileOutputStream(dst), StandardCharsets.UTF_8);
for (String line = in.readLine(); null != line; line = in.readLine()) {
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminCreateDiscoverTest.java b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminCreateDiscoverTest.java
index 8b97f7124ad..33347f9a01d 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminCreateDiscoverTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminCreateDiscoverTest.java
@@ -24,7 +24,6 @@ import java.io.InputStreamReader;
import java.util.Properties;
import org.apache.commons.io.FileUtils;
-import org.apache.lucene.util.IOUtils;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CoreAdminParams;
@@ -35,6 +34,8 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.nio.charset.StandardCharsets;
+
public class CoreAdminCreateDiscoverTest extends SolrTestCaseJ4 {
private static File solrHomeDirectory = null;
@@ -108,7 +109,7 @@ public class CoreAdminCreateDiscoverTest extends SolrTestCaseJ4 {
File propFile = new File(solrHomeDirectory, coreSysProps + "/" + CorePropertiesLocator.PROPERTIES_FILENAME);
FileInputStream is = new FileInputStream(propFile);
try {
- props.load(new InputStreamReader(is, IOUtils.CHARSET_UTF_8));
+ props.load(new InputStreamReader(is, StandardCharsets.UTF_8));
} finally {
org.apache.commons.io.IOUtils.closeQuietly(is);
}
@@ -250,7 +251,7 @@ public class CoreAdminCreateDiscoverTest extends SolrTestCaseJ4 {
File propFile = new File(solrHomeDirectory, coreNormal + "/" + CorePropertiesLocator.PROPERTIES_FILENAME);
FileInputStream is = new FileInputStream(propFile);
try {
- props.load(new InputStreamReader(is, IOUtils.CHARSET_UTF_8));
+ props.load(new InputStreamReader(is, StandardCharsets.UTF_8));
} finally {
org.apache.commons.io.IOUtils.closeQuietly(is);
}
diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedExpandComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedExpandComponentTest.java
index d328bb82389..9abf451c5b9 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/DistributedExpandComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedExpandComponentTest.java
@@ -88,6 +88,9 @@ public class DistributedExpandComponentTest extends BaseDistributedSearchTestCas
query("q", "test_ti:5", "fq", "{!collapse field=group_s}", "defType", "edismax", "bf", "field(test_ti)", "expand", "true", "expand.sort", "test_tl desc", "expand.rows", "1", "fl","*,score");
//Test zero results
query("q", "test_ti:5434343", "fq", "{!collapse field=group_s}", "defType", "edismax", "bf", "field(test_ti)", "expand", "true", "expand.sort", "test_tl desc", "expand.rows", "1", "fl","*,score");
+ //Test page 2
+ query("q", "*:*", "start","1", "rows", "1", "fq", "{!collapse field=group_s}", "defType", "edismax", "bf", "field(test_ti)", "expand", "true", "fl","*,score");
+
//First basic test case.
ModifiableSolrParams params = new ModifiableSolrParams();
diff --git a/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java
index f9da1725f8d..a26ee20d7b5 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java
@@ -19,7 +19,6 @@ package org.apache.solr.handler.component;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.GroupParams;
@@ -38,6 +37,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
@@ -674,7 +674,7 @@ public class QueryElevationComponentTest extends SolrTestCaseJ4 {
// write a test file to boost some docs
private void writeFile(File file, String query, String... ids) throws Exception {
- PrintWriter out = new PrintWriter(new OutputStreamWriter(new FileOutputStream(file), IOUtils.CHARSET_UTF_8));
+ PrintWriter out = new PrintWriter(new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8));
out.println("");
out.println("");
out.println("");
diff --git a/solr/core/src/test/org/apache/solr/handler/component/TestExpandComponent.java b/solr/core/src/test/org/apache/solr/handler/component/TestExpandComponent.java
index 792403e96ff..ea55c78c187 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/TestExpandComponent.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/TestExpandComponent.java
@@ -45,10 +45,10 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
@Test
public void testExpand() throws Exception {
- String[] doc = {"id","1", "term_s", "YYYY", "group_s", "group1", "test_ti", "5", "test_tl", "10", "test_tf", "2000"};
+ String[] doc = {"id","1", "term_s", "YYYY", "group_s", "group1", "test_ti", "5", "test_tl", "10", "test_tf", "2000", "type_s", "parent"};
assertU(adoc(doc));
assertU(commit());
- String[] doc1 = {"id","2", "term_s","YYYY", "group_s", "group1", "test_ti", "50", "test_tl", "100", "test_tf", "200"};
+ String[] doc1 = {"id","2", "term_s","YYYY", "group_s", "group1", "test_ti", "50", "test_tl", "100", "test_tf", "200", "type_s", "child"};
assertU(adoc(doc1));
String[] doc2 = {"id","3", "term_s", "YYYY", "test_ti", "5000", "test_tl", "100", "test_tf", "200"};
@@ -58,23 +58,21 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
assertU(adoc(doc3));
- String[] doc4 = {"id","5", "term_s", "YYYY", "group_s", "group2", "test_ti", "4", "test_tl", "10", "test_tf", "2000"};
+ String[] doc4 = {"id","5", "term_s", "YYYY", "group_s", "group2", "test_ti", "4", "test_tl", "10", "test_tf", "2000", "type_s", "parent"};
assertU(adoc(doc4));
assertU(commit());
- String[] doc5 = {"id","6", "term_s","YYYY", "group_s", "group2", "test_ti", "10", "test_tl", "100", "test_tf", "200"};
+ String[] doc5 = {"id","6", "term_s","YYYY", "group_s", "group2", "test_ti", "10", "test_tl", "100", "test_tf", "200", "type_s", "child"};
assertU(adoc(doc5));
assertU(commit());
- String[] doc6 = {"id","7", "term_s", "YYYY", "group_s", "group1", "test_ti", "1", "test_tl", "100000", "test_tf", "2000"};
+ String[] doc6 = {"id","7", "term_s", "YYYY", "group_s", "group1", "test_ti", "1", "test_tl", "100000", "test_tf", "2000", "type_s", "child"};
assertU(adoc(doc6));
assertU(commit());
- String[] doc7 = {"id","8", "term_s","YYYY", "group_s", "group2", "test_ti", "2", "test_tl", "100000", "test_tf", "200"};
+ String[] doc7 = {"id","8", "term_s","YYYY", "group_s", "group2", "test_ti", "2", "test_tl", "100000", "test_tf", "200", "type_s", "child"};
assertU(adoc(doc7));
assertU(commit());
-
-
//First basic test case.
ModifiableSolrParams params = new ModifiableSolrParams();
params.add("q", "*:*");
@@ -92,6 +90,23 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
"/response/lst[@name='expanded']/result[@name='group2']/doc[2]/float[@name='id'][.='8.0']"
);
+ //Basic test case page 2
+
+ params = new ModifiableSolrParams();
+ params.add("q", "*:*");
+ params.add("fq", "{!collapse field=group_s}");
+ params.add("defType", "edismax");
+ params.add("bf", "field(test_ti)");
+ params.add("expand", "true");
+ params.add("rows", "1");
+ params.add("start", "1");
+ assertQ(req(params), "*[count(/response/result/doc)=1]",
+ "*[count(/response/lst[@name='expanded']/result)=1]",
+ "/response/result/doc[1]/float[@name='id'][.='6.0']",
+ "/response/lst[@name='expanded']/result[@name='group2']/doc[1]/float[@name='id'][.='5.0']",
+ "/response/lst[@name='expanded']/result[@name='group2']/doc[2]/float[@name='id'][.='8.0']"
+ );
+
//Test expand.sort
params = new ModifiableSolrParams();
params.add("q", "*:*");
@@ -131,6 +146,70 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
);
+ //Test overide expand.q
+
+ params = new ModifiableSolrParams();
+ params.add("q", "type_s:parent");
+ params.add("defType", "edismax");
+ params.add("bf", "field(test_ti)");
+ params.add("expand", "true");
+ params.add("expand.q", "type_s:child");
+ params.add("expand.field", "group_s");
+ params.add("expand.sort", "test_tl desc");
+ assertQ(req(params), "*[count(/response/result/doc)=2]",
+ "*[count(/response/lst[@name='expanded']/result)=2]",
+ "/response/result/doc[1]/float[@name='id'][.='1.0']",
+ "/response/result/doc[2]/float[@name='id'][.='5.0']",
+ "/response/lst[@name='expanded']/result[@name='group1']/doc[1]/float[@name='id'][.='7.0']",
+ "/response/lst[@name='expanded']/result[@name='group1']/doc[2]/float[@name='id'][.='2.0']",
+ "/response/lst[@name='expanded']/result[@name='group2']/doc[1]/float[@name='id'][.='8.0']",
+ "/response/lst[@name='expanded']/result[@name='group2']/doc[2]/float[@name='id'][.='6.0']"
+ );
+
+
+ //Test overide expand.fq
+
+ params = new ModifiableSolrParams();
+ params.add("q", "*:*");
+ params.add("fq", "type_s:parent");
+ params.add("defType", "edismax");
+ params.add("bf", "field(test_ti)");
+ params.add("expand", "true");
+ params.add("expand.fq", "type_s:child");
+ params.add("expand.field", "group_s");
+ params.add("expand.sort", "test_tl desc");
+ assertQ(req(params), "*[count(/response/result/doc)=2]",
+ "*[count(/response/lst[@name='expanded']/result)=2]",
+ "/response/result/doc[1]/float[@name='id'][.='1.0']",
+ "/response/result/doc[2]/float[@name='id'][.='5.0']",
+ "/response/lst[@name='expanded']/result[@name='group1']/doc[1]/float[@name='id'][.='7.0']",
+ "/response/lst[@name='expanded']/result[@name='group1']/doc[2]/float[@name='id'][.='2.0']",
+ "/response/lst[@name='expanded']/result[@name='group2']/doc[1]/float[@name='id'][.='8.0']",
+ "/response/lst[@name='expanded']/result[@name='group2']/doc[2]/float[@name='id'][.='6.0']"
+ );
+
+ //Test overide expand.fq and expand.q
+
+ params = new ModifiableSolrParams();
+ params.add("q", "*:*");
+ params.add("fq", "type_s:parent");
+ params.add("defType", "edismax");
+ params.add("bf", "field(test_ti)");
+ params.add("expand", "true");
+ params.add("expand.q", "type_s:child");
+ params.add("expand.fq", "*:*");
+ params.add("expand.field", "group_s");
+ params.add("expand.sort", "test_tl desc");
+ assertQ(req(params), "*[count(/response/result/doc)=2]",
+ "*[count(/response/lst[@name='expanded']/result)=2]",
+ "/response/result/doc[1]/float[@name='id'][.='1.0']",
+ "/response/result/doc[2]/float[@name='id'][.='5.0']",
+ "/response/lst[@name='expanded']/result[@name='group1']/doc[1]/float[@name='id'][.='7.0']",
+ "/response/lst[@name='expanded']/result[@name='group1']/doc[2]/float[@name='id'][.='2.0']",
+ "/response/lst[@name='expanded']/result[@name='group2']/doc[1]/float[@name='id'][.='8.0']",
+ "/response/lst[@name='expanded']/result[@name='group2']/doc[2]/float[@name='id'][.='6.0']"
+ );
+
//Test expand.rows
params = new ModifiableSolrParams();
@@ -179,15 +258,6 @@ public class TestExpandComponent extends SolrTestCaseJ4 {
assertQ(req(params), "*[count(/response/result/doc)=0]",
"*[count(/response/lst[@name='expanded']/result)=0]"
);
-
-
-
-
-
-
}
-
-
-
}
diff --git a/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java b/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java
index 102533930c8..ec41fe1a47a 100644
--- a/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java
+++ b/solr/core/src/test/org/apache/solr/highlight/TestPostingsSolrHighlighter.java
@@ -24,7 +24,6 @@ import org.apache.solr.schema.IndexSchema;
import org.junit.BeforeClass;
/** simple tests for PostingsSolrHighlighter */
-@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom"})
public class TestPostingsSolrHighlighter extends SolrTestCaseJ4 {
@BeforeClass
diff --git a/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java b/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java
index 37ad252cfc0..9135bc40533 100644
--- a/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java
+++ b/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java
@@ -19,6 +19,7 @@
package org.apache.solr.internal.csv.writer;
import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
import junit.framework.TestCase;
@@ -57,7 +58,7 @@ public class CSVConfigGuesserTest extends TestCase {
StringBuilder sb = new StringBuilder();
sb.append("1234;abcd;1234\n");
sb.append("abcd;1234;abcd");
- ByteArrayInputStream in = new ByteArrayInputStream(sb.toString().getBytes("UTF-8"));
+ ByteArrayInputStream in = new ByteArrayInputStream(sb.toString().getBytes(StandardCharsets.UTF_8));
CSVConfigGuesser guesser = new CSVConfigGuesser(in);
CSVConfig guessed = guesser.guess();
assertEquals(expected.isFixedWidth(), guessed.isFixedWidth());
@@ -80,7 +81,7 @@ public class CSVConfigGuesserTest extends TestCase {
StringBuilder sb = new StringBuilder();
sb.append("1,2,3,4\n");
sb.append("abcd,1234,abcd,1234");
- ByteArrayInputStream in = new ByteArrayInputStream(sb.toString().getBytes("UTF-8"));
+ ByteArrayInputStream in = new ByteArrayInputStream(sb.toString().getBytes(StandardCharsets.UTF_8));
CSVConfigGuesser guesser = new CSVConfigGuesser(in);
CSVConfig guessed = guesser.guess();
assertEquals(expected.isFixedWidth(), guessed.isFixedWidth());
diff --git a/solr/core/src/test/org/apache/solr/request/JSONWriterTest.java b/solr/core/src/test/org/apache/solr/request/JSONWriterTest.java
index 84350beb237..9d2ca2ee1c9 100644
--- a/solr/core/src/test/org/apache/solr/request/JSONWriterTest.java
+++ b/solr/core/src/test/org/apache/solr/request/JSONWriterTest.java
@@ -19,6 +19,7 @@ package org.apache.solr.request;
import java.io.IOException;
import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
import java.util.HashSet;
import java.util.Set;
@@ -92,7 +93,7 @@ public class JSONWriterTest extends SolrTestCaseJ4 {
rsp.add("byte", Byte.valueOf((byte)-3));
rsp.add("short", Short.valueOf((short)-4));
- rsp.add("bytes", "abc".getBytes("UTF-8"));
+ rsp.add("bytes", "abc".getBytes(StandardCharsets.UTF_8));
w.write(buf, req, rsp);
jsonEq("{\"nl\":[[\"data1\",\"he\\u2028llo\\u2029!\"],[null,42]],\"byte\":-3,\"short\":-4,\"bytes\":\"YWJj\"}", buf.toString());
diff --git a/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java b/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java
index 14f88271e72..fce5d7871de 100644
--- a/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java
+++ b/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java
@@ -44,6 +44,7 @@ import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
/**
* See SOLR-2854.
@@ -99,7 +100,7 @@ public class TestRemoteStreaming extends SolrJettyTestBase {
InputStream inputStream = (InputStream) obj;
try {
StringWriter strWriter = new StringWriter();
- IOUtils.copy(new InputStreamReader(inputStream, "UTF-8"),strWriter);
+ IOUtils.copy(new InputStreamReader(inputStream, StandardCharsets.UTF_8),strWriter);
return strWriter.toString();
} finally {
IOUtils.closeQuietly(inputStream);
diff --git a/solr/core/src/test/org/apache/solr/request/TestWriterPerf.java b/solr/core/src/test/org/apache/solr/request/TestWriterPerf.java
index d9a760f7f97..83b91491127 100644
--- a/solr/core/src/test/org/apache/solr/request/TestWriterPerf.java
+++ b/solr/core/src/test/org/apache/solr/request/TestWriterPerf.java
@@ -21,6 +21,7 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import org.apache.solr.client.solrj.ResponseParser;
@@ -119,7 +120,7 @@ public class TestWriterPerf extends AbstractSolrTestCase {
out = new ByteArrayOutputStream();
// to be fair, from my previous tests, much of the performance will be sucked up
// by java's UTF-8 encoding/decoding, not the actual writing
- Writer writer = new OutputStreamWriter(out, "UTF-8");
+ Writer writer = new OutputStreamWriter(out, StandardCharsets.UTF_8);
w.write(writer, req, rsp);
writer.close();
}
diff --git a/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymFilterFactory.java b/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymFilterFactory.java
new file mode 100644
index 00000000000..1c91ab9142e
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymFilterFactory.java
@@ -0,0 +1,178 @@
+package org.apache.solr.rest.schema.analysis;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.solr.util.RestTestBase;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.noggit.JSONUtil;
+import org.restlet.ext.servlet.ServerServlet;
+
+public class TestManagedSynonymFilterFactory extends RestTestBase {
+
+ private static File tmpSolrHome;
+
+ /**
+ * Setup to make the schema mutable
+ */
+ @Before
+ public void before() throws Exception {
+ tmpSolrHome = createTempDir();
+ FileUtils.copyDirectory(new File(TEST_HOME()), tmpSolrHome.getAbsoluteFile());
+
+ final SortedMap extraServlets = new TreeMap<>();
+ final ServletHolder solrRestApi = new ServletHolder("SolrSchemaRestApi", ServerServlet.class);
+ solrRestApi.setInitParameter("org.restlet.application", "org.apache.solr.rest.SolrSchemaRestApi");
+ extraServlets.put(solrRestApi, "/schema/*");
+
+ System.setProperty("managed.schema.mutable", "true");
+ System.setProperty("enable.update.log", "false");
+ createJettyAndHarness(tmpSolrHome.getAbsolutePath(), "solrconfig-managed-schema.xml", "schema-rest.xml",
+ "/solr", true, extraServlets);
+ }
+
+ @After
+ private void after() throws Exception {
+ jetty.stop();
+ jetty = null;
+ FileUtils.deleteDirectory(tmpSolrHome);
+ System.clearProperty("managed.schema.mutable");
+ System.clearProperty("enable.update.log");
+ }
+
+ @Test
+ public void testManagedSynonyms() throws Exception {
+ // this endpoint depends on at least one field type containing the following
+ // declaration in the schema-rest.xml:
+ //
+ // > syns = new HashMap<>();
+ syns.put("happy", Arrays.asList("glad","cheerful","joyful"));
+ assertJPut(endpoint,
+ JSONUtil.toJSON(syns),
+ "/responseHeader/status==0");
+
+ assertJQ(endpoint,
+ "/synonymMappings/managedMap/happy==['cheerful','glad','joyful']");
+
+ // request to a specific mapping
+ assertJQ(endpoint+"/happy",
+ "/happy==['cheerful','glad','joyful']");
+
+ // does not exist
+ assertJQ(endpoint+"/sad",
+ "/error/code==404");
+
+ // verify the user can update the ignoreCase initArg
+ assertJPut(endpoint,
+ json("{ 'initArgs':{ 'ignoreCase':true } }"),
+ "responseHeader/status==0");
+
+ assertJQ(endpoint,
+ "/synonymMappings/initArgs/ignoreCase==true");
+
+ syns = new HashMap<>();
+ syns.put("sad", Arrays.asList("unhappy"));
+ syns.put("SAD", Arrays.asList("Unhappy"));
+ assertJPut(endpoint,
+ JSONUtil.toJSON(syns),
+ "/responseHeader/status==0");
+
+ assertJQ(endpoint,
+ "/synonymMappings/managedMap/sad==['unhappy']");
+
+ // verify delete works
+ assertJDelete(endpoint+"/sad",
+ "/responseHeader/status==0");
+
+ assertJQ(endpoint,
+ "/synonymMappings/managedMap=={'happy':['cheerful','glad','joyful']}");
+
+ // should fail with 404 as foo doesn't exist
+ assertJDelete(endpoint+"/foo",
+ "/error/code==404");
+
+ // verify that a newly added synonym gets expanded on the query side after core reload
+
+ String newFieldName = "managed_en_field";
+ // make sure the new field doesn't already exist
+ assertQ("/schema/fields/" + newFieldName + "?indent=on&wt=xml",
+ "count(/response/lst[@name='field']) = 0",
+ "/response/lst[@name='responseHeader']/int[@name='status'] = '404'",
+ "/response/lst[@name='error']/int[@name='code'] = '404'");
+
+ // add the new field
+ assertJPut("/schema/fields/" + newFieldName, json("{'type':'managed_en'}"),
+ "/responseHeader/status==0");
+
+ // make sure the new field exists now
+ assertQ("/schema/fields/" + newFieldName + "?indent=on&wt=xml",
+ "count(/response/lst[@name='field']) = 1",
+ "/response/lst[@name='responseHeader']/int[@name='status'] = '0'");
+
+ assertU(adoc(newFieldName, "I am a happy test today but yesterday I was angry", "id", "5150"));
+ assertU(commit());
+
+ assertQ("/select?q=" + newFieldName + ":angry",
+ "/response/lst[@name='responseHeader']/int[@name='status'] = '0'",
+ "/response/result[@name='response'][@numFound='1']",
+ "/response/result[@name='response']/doc/str[@name='id'][.='5150']");
+
+ // add a mapping that will expand a query for "mad" to match docs with "angry"
+ syns = new HashMap<>();
+ syns.put("mad", Arrays.asList("angry"));
+ assertJPut(endpoint,
+ JSONUtil.toJSON(syns),
+ "/responseHeader/status==0");
+
+ assertJQ(endpoint,
+ "/synonymMappings/managedMap/mad==['angry']");
+
+ // should not match as the synonym mapping between mad and angry does not
+ // get applied until core reload
+ assertQ("/select?q=" + newFieldName + ":mad",
+ "/response/lst[@name='responseHeader']/int[@name='status'] = '0'",
+ "/response/result[@name='response'][@numFound='0']");
+
+ restTestHarness.reload();
+
+ // now query for mad and we should see our test doc
+ assertQ("/select?q=" + newFieldName + ":mad",
+ "/response/lst[@name='responseHeader']/int[@name='status'] = '0'",
+ "/response/result[@name='response'][@numFound='1']",
+ "/response/result[@name='response']/doc/str[@name='id'][.='5150']");
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldTest.java b/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldTest.java
index f7135f46302..f30b7ba96bb 100644
--- a/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldTest.java
+++ b/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldTest.java
@@ -72,7 +72,7 @@ public class PreAnalyzedFieldTest extends SolrTestCaseJ4 {
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig.xml","schema.xml");
+ initCore("solrconfig-minimal.xml","schema-preanalyzed.xml");
}
@Override
@@ -101,6 +101,12 @@ public class PreAnalyzedFieldTest extends SolrTestCaseJ4 {
}
}
}
+
+ @Test
+ public void testValidSimple2() {
+ assertU(adoc("id", "1",
+ "pre", "{\"v\":\"1\",\"str\":\"document one\",\"tokens\":[{\"t\":\"one\"},{\"t\":\"two\"},{\"t\":\"three\",\"i\":100}]}"));
+ }
@Test
public void testInvalidSimple() {
diff --git a/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java b/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java
index ea423abca59..a01dbd186bd 100644
--- a/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java
+++ b/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java
@@ -24,7 +24,6 @@ import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.FieldComparatorSource;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
-import org.apache.solr.common.util.Base64;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -81,20 +80,11 @@ public class SortableBinaryField extends BinaryField {
@Override
public Object marshalSortValue(Object value) {
- if (null == value) {
- return null;
- }
- final BytesRef val = (BytesRef)value;
- return Base64.byteArrayToBase64(val.bytes, val.offset, val.length);
+ return marshalBase64SortValue(value);
}
@Override
public Object unmarshalSortValue(Object value) {
- if (null == value) {
- return null;
- }
- final String val = (String)value;
- final byte[] bytes = Base64.base64ToByteArray(val);
- return new BytesRef(bytes);
+ return unmarshalBase64SortValue(value);
}
}
diff --git a/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchema.java b/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchema.java
index 6deaa45b526..15422d1727f 100644
--- a/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchema.java
+++ b/solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchema.java
@@ -29,6 +29,7 @@ import org.apache.zookeeper.KeeperException;
import org.junit.BeforeClass;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.List;
public class TestCloudManagedSchema extends AbstractFullDistribZkTestBase {
@@ -89,14 +90,14 @@ public class TestCloudManagedSchema extends AbstractFullDistribZkTestBase {
private String getFileContentFromZooKeeper(SolrZkClient zkClient, String fileName)
throws IOException, SolrServerException, KeeperException, InterruptedException {
- return (new String(zkClient.getData(fileName, null, null, true), "UTF-8"));
+ return (new String(zkClient.getData(fileName, null, null, true), StandardCharsets.UTF_8));
}
protected final void assertFileNotInZooKeeper(SolrZkClient zkClient, String parent, String fileName) throws Exception {
List kids = zkClient.getChildren(parent, null, true);
for (String kid : kids) {
if (kid.equalsIgnoreCase(fileName)) {
- String rawContent = new String(zkClient.getData(fileName, null, null, true), "UTF-8");
+ String rawContent = new String(zkClient.getData(fileName, null, null, true), StandardCharsets.UTF_8);
fail("File '" + fileName + "' was unexpectedly found in ZooKeeper. Content starts with '"
+ rawContent.substring(0, 100) + " [...]'");
}
diff --git a/solr/core/src/test/org/apache/solr/search/CursorMarkTest.java b/solr/core/src/test/org/apache/solr/search/CursorMarkTest.java
index 950d9366993..61799eb1522 100644
--- a/solr/core/src/test/org/apache/solr/search/CursorMarkTest.java
+++ b/solr/core/src/test/org/apache/solr/search/CursorMarkTest.java
@@ -17,10 +17,14 @@
package org.apache.solr.search;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.schema.DateField;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.request.SolrQueryRequest;
@@ -28,11 +32,14 @@ import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.CursorPagingTest;
import static org.apache.solr.common.params.CursorMarkParams.CURSOR_MARK_START;
+import java.io.IOException;
import java.util.Arrays;
import java.util.ArrayList;
+import java.util.Date;
import java.util.List;
import java.util.Collection;
import java.util.Collections;
+import java.util.UUID;
import org.junit.BeforeClass;
@@ -51,7 +58,7 @@ public class CursorMarkTest extends SolrTestCaseJ4 {
initCore(CursorPagingTest.TEST_SOLRCONFIG_NAME, CursorPagingTest.TEST_SCHEMAXML_NAME);
}
- public void testNextCursorMark() {
+ public void testNextCursorMark() throws IOException {
final Collection allFieldNames = getAllFieldNames();
final SolrQueryRequest req = req();
final IndexSchema schema = req.getSchema();
@@ -113,7 +120,7 @@ public class CursorMarkTest extends SolrTestCaseJ4 {
}
- public void testGarbageParsing() {
+ public void testGarbageParsing() throws IOException {
final SolrQueryRequest req = req();
final IndexSchema schema = req.getSchema();
final SortSpec ss = QueryParsing.parseSortSpec("str asc, float desc, id asc", req);
@@ -160,7 +167,7 @@ public class CursorMarkTest extends SolrTestCaseJ4 {
}
}
- public void testRoundTripParsing() {
+ public void testRoundTripParsing() throws IOException {
// for any valid SortSpec, and any legal values, we should be able to round
// trip serialize the totem and get the same values back.
@@ -196,7 +203,7 @@ public class CursorMarkTest extends SolrTestCaseJ4 {
}
}
- private static Object[] buildRandomSortObjects(SortSpec ss) {
+ private static Object[] buildRandomSortObjects(SortSpec ss) throws IOException {
List fields = ss.getSchemaFields();
assertNotNull(fields);
Object[] results = new Object[fields.size()];
@@ -225,14 +232,64 @@ public class CursorMarkTest extends SolrTestCaseJ4 {
byte[] randBytes = new byte[TestUtil.nextInt(random(), 1, 50)];
random().nextBytes(randBytes);
val = new BytesRef(randBytes);
- } else if (fieldName.startsWith("int")) {
- val = (Integer) random().nextInt();
- } else if (fieldName.startsWith("long")) {
- val = (Long) random().nextLong();
- } else if (fieldName.startsWith("float")) {
- val = (Float) random().nextFloat() * random().nextInt(); break;
- } else if (fieldName.startsWith("double")) {
- val = (Double) random().nextDouble() * random().nextInt(); break;
+ } else if (fieldName.startsWith("bcd")) {
+ if (fieldName.startsWith("bcd_long")) { // BCDLongField
+ val = Long.toString(random().nextLong());
+ val = sf.getType().toInternal((String)val);
+ val = sf.getType().unmarshalSortValue(val);
+ } else { // BCDIntField & BCDStrField
+ val = Integer.toString(random().nextInt());
+ val = sf.getType().toInternal((String)val);
+ val = sf.getType().unmarshalSortValue(val);
+ }
+ } else if (fieldName.contains("int")) {
+ val = random().nextInt(); // TrieIntField
+ if (fieldName.startsWith("legacy")) { // IntField
+ val = Integer.toString((Integer)val);
+ if (fieldName.startsWith("legacy_sortable")) { // SortableIntField
+ val = sf.getType().unmarshalSortValue(val);
+ }
+ }
+ } else if (fieldName.contains("long")) {
+ val = random().nextLong(); // TrieLongField
+ if (fieldName.startsWith("legacy")) { // LongField
+ val = Long.toString((Long)val);
+ if (fieldName.startsWith("legacy_sortable")) { // SortableLongField
+ val = sf.getType().unmarshalSortValue(val);
+ }
+ }
+ } else if (fieldName.contains("float")) {
+ val = random().nextFloat() * random().nextInt(); // TrieFloatField
+ if (fieldName.startsWith("legacy")) { // FloatField
+ val = Float.toString((Float)val);
+ if (fieldName.startsWith("legacy_sortable")) { // SortableFloatField
+ val = sf.getType().unmarshalSortValue(val);
+ }
+ }
+ } else if (fieldName.contains("double")) {
+ val = random().nextDouble() * random().nextInt(); // TrieDoubleField
+ if (fieldName.startsWith("legacy")) { // DoubleField
+ val = Double.toString((Double)val);
+ if (fieldName.startsWith("legacy_sortable")) { // SortableDoubleField
+ val = sf.getType().unmarshalSortValue(val);
+ }
+ }
+ } else if (fieldName.contains("date")) {
+ val = random().nextLong(); // TrieDateField
+ if (fieldName.startsWith("legacy_date")) { // DateField
+ val = ((DateField)sf.getType()).toInternal(new Date((Long)val));
+ val = sf.getType().unmarshalSortValue(val);
+ }
+ } else if (fieldName.startsWith("currency")) {
+ val = random().nextDouble();
+ } else if (fieldName.startsWith("uuid")) {
+ val = sf.getType().unmarshalSortValue(UUID.randomUUID().toString());
+ } else if (fieldName.startsWith("bool")) {
+ val = sf.getType().unmarshalSortValue(random().nextBoolean() ? "t" : "f");
+ } else if (fieldName.startsWith("enum")) {
+ val = random().nextInt(CursorPagingTest.SEVERITY_ENUM_VALUES.length);
+ } else if (fieldName.contains("collation")) {
+ val = getRandomCollation(sf);
} else {
fail("fell through the rabbit hole, new field in schema? = " + fieldName);
}
@@ -243,6 +300,22 @@ public class CursorMarkTest extends SolrTestCaseJ4 {
}
return results;
}
+
+ private static Object getRandomCollation(SchemaField sf) throws IOException {
+ Object val;
+ Analyzer analyzer = sf.getType().getAnalyzer();
+ String term = TestUtil.randomRealisticUnicodeString(random());
+ try (TokenStream ts = analyzer.tokenStream("fake", term)) {
+ TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
+ val = termAtt.getBytesRef();
+ ts.reset();
+ assertTrue(ts.incrementToken());
+ termAtt.fillBytesRef();
+ assertFalse(ts.incrementToken());
+ ts.end();
+ }
+ return val;
+ }
/**
* a list of the fields in the schema - excluding _version_
diff --git a/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java
index cf34e74e833..195ff656ca3 100644
--- a/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java
+++ b/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java
@@ -23,9 +23,11 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import com.carrotsearch.hppc.IntOpenHashSet;
import java.io.IOException;
import java.util.*;
+import java.util.Random;
public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
@@ -146,6 +148,51 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
"//result/doc[3]/float[@name='id'][.='3.0']",
"//result/doc[4]/float[@name='id'][.='6.0']");
+ //Test SOLR-5773 with score collapse criteria
+ params = new ModifiableSolrParams();
+ params.add("q", "YYYY");
+ params.add("fq", "{!collapse field=group_s nullPolicy=collapse}");
+ params.add("defType", "edismax");
+ params.add("bf", "field(test_ti)");
+ params.add("qf", "term_s");
+ params.add("qt", "/elevate");
+ params.add("elevateIds", "1,5");
+ assertQ(req(params), "*[count(//doc)=3]",
+ "//result/doc[1]/float[@name='id'][.='1.0']",
+ "//result/doc[2]/float[@name='id'][.='5.0']",
+ "//result/doc[3]/float[@name='id'][.='3.0']");
+
+ //Test SOLR-5773 with max field collapse criteria
+ params = new ModifiableSolrParams();
+ params.add("q", "YYYY");
+ params.add("fq", "{!collapse field=group_s min=test_ti nullPolicy=collapse}");
+ params.add("defType", "edismax");
+ params.add("bf", "field(test_ti)");
+ params.add("qf", "term_s");
+ params.add("qt", "/elevate");
+ params.add("elevateIds", "1,5");
+ assertQ(req(params), "*[count(//doc)=3]",
+ "//result/doc[1]/float[@name='id'][.='1.0']",
+ "//result/doc[2]/float[@name='id'][.='5.0']",
+ "//result/doc[3]/float[@name='id'][.='4.0']");
+
+
+ //Test SOLR-5773 elevating documents with null group
+ params = new ModifiableSolrParams();
+ params.add("q", "YYYY");
+ params.add("fq", "{!collapse field=group_s}");
+ params.add("defType", "edismax");
+ params.add("bf", "field(test_ti)");
+ params.add("qf", "term_s");
+ params.add("qt", "/elevate");
+ params.add("elevateIds", "3,4");
+ assertQ(req(params), "*[count(//doc)=4]",
+ "//result/doc[1]/float[@name='id'][.='3.0']",
+ "//result/doc[2]/float[@name='id'][.='4.0']",
+ "//result/doc[3]/float[@name='id'][.='2.0']",
+ "//result/doc[4]/float[@name='id'][.='6.0']");
+
+
//Test collapse by min int field and sort
params = new ModifiableSolrParams();
diff --git a/solr/core/src/test/org/apache/solr/search/TestDocSet.java b/solr/core/src/test/org/apache/solr/search/TestDocSet.java
index 2bf35dca533..af588e11c33 100644
--- a/solr/core/src/test/org/apache/solr/search/TestDocSet.java
+++ b/solr/core/src/test/org/apache/solr/search/TestDocSet.java
@@ -418,6 +418,10 @@ public class TestDocSet extends LuceneTestCase {
@Override
public void document(int doc, StoredFieldVisitor visitor) {
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ }
};
}
diff --git a/solr/core/src/test/org/apache/solr/search/TestRecovery.java b/solr/core/src/test/org/apache/solr/search/TestRecovery.java
index 1b8bc633cd9..95f20171ab5 100644
--- a/solr/core/src/test/org/apache/solr/search/TestRecovery.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRecovery.java
@@ -33,6 +33,7 @@ import org.junit.Test;
import java.io.File;
import java.io.RandomAccessFile;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayDeque;
import java.util.Arrays;
import java.util.Deque;
@@ -1030,9 +1031,9 @@ public class TestRecovery extends SolrTestCaseJ4 {
raf.close();
// Now make a newer log file with just the IDs changed. NOTE: this may not work if log format changes too much!
- findReplace("AAAAAA".getBytes("UTF-8"), "aaaaaa".getBytes("UTF-8"), content);
- findReplace("BBBBBB".getBytes("UTF-8"), "bbbbbb".getBytes("UTF-8"), content);
- findReplace("CCCCCC".getBytes("UTF-8"), "cccccc".getBytes("UTF-8"), content);
+ findReplace("AAAAAA".getBytes(StandardCharsets.UTF_8), "aaaaaa".getBytes(StandardCharsets.UTF_8), content);
+ findReplace("BBBBBB".getBytes(StandardCharsets.UTF_8), "bbbbbb".getBytes(StandardCharsets.UTF_8), content);
+ findReplace("CCCCCC".getBytes(StandardCharsets.UTF_8), "cccccc".getBytes(StandardCharsets.UTF_8), content);
// WARNING... assumes format of .00000n where n is less than 9
long logNumber = Long.parseLong(fname.substring(fname.lastIndexOf(".") + 1));
diff --git a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
index 8386ab1ff86..dfe64bc0299 100644
--- a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
@@ -23,6 +23,7 @@ import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayDeque;
import java.util.Arrays;
import java.util.Deque;
@@ -1028,9 +1029,9 @@ public class TestRecoveryHdfs extends SolrTestCaseJ4 {
dis.close();
// Now make a newer log file with just the IDs changed. NOTE: this may not work if log format changes too much!
- findReplace("AAAAAA".getBytes("UTF-8"), "aaaaaa".getBytes("UTF-8"), content);
- findReplace("BBBBBB".getBytes("UTF-8"), "bbbbbb".getBytes("UTF-8"), content);
- findReplace("CCCCCC".getBytes("UTF-8"), "cccccc".getBytes("UTF-8"), content);
+ findReplace("AAAAAA".getBytes(StandardCharsets.UTF_8), "aaaaaa".getBytes(StandardCharsets.UTF_8), content);
+ findReplace("BBBBBB".getBytes(StandardCharsets.UTF_8), "bbbbbb".getBytes(StandardCharsets.UTF_8), content);
+ findReplace("CCCCCC".getBytes(StandardCharsets.UTF_8), "cccccc".getBytes(StandardCharsets.UTF_8), content);
// WARNING... assumes format of .00000n where n is less than 9
long logNumber = Long.parseLong(fname.substring(fname.lastIndexOf(".") + 1));
diff --git a/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java b/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
index 7f1bba79bbc..3952fb09703 100644
--- a/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
+++ b/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
@@ -28,9 +28,11 @@ import org.apache.solr.common.util.NamedList;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.Ignore;
+
import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -49,11 +51,12 @@ public class TestFunctionQuery extends SolrTestCaseJ4 {
String base = "external_foo_extf";
static long start = System.currentTimeMillis();
- void makeExternalFile(String field, String contents, String charset) {
+
+ void makeExternalFile(String field, String contents) {
String dir = h.getCore().getDataDir();
String filename = dir + "/external_" + field + "." + (start++);
try {
- Writer out = new OutputStreamWriter(new FileOutputStream(filename), charset);
+ Writer out = new OutputStreamWriter(new FileOutputStream(filename), StandardCharsets.UTF_8);
out.write(contents);
out.close();
} catch (Exception e) {
@@ -219,7 +222,7 @@ public class TestFunctionQuery extends SolrTestCaseJ4 {
createIndex(null,ids);
// Unsorted field, largest first
- makeExternalFile(field, "54321=543210\n0=-999\n25=250","UTF-8");
+ makeExternalFile(field, "54321=543210\n0=-999\n25=250");
// test identity (straight field value)
singleTest(field, "\0", 54321, 543210, 0,-999, 25,250, 100, 1);
Object orig = FileFloatSource.onlyForTesting;
@@ -229,7 +232,7 @@ public class TestFunctionQuery extends SolrTestCaseJ4 {
singleTest(field, "sqrt(\0)");
assertTrue(orig == FileFloatSource.onlyForTesting);
- makeExternalFile(field, "0=1","UTF-8");
+ makeExternalFile(field, "0=1");
assertU(h.query("/reloadCache",lrf.makeRequest("","")));
singleTest(field, "sqrt(\0)");
assertTrue(orig != FileFloatSource.onlyForTesting);
@@ -263,7 +266,7 @@ public class TestFunctionQuery extends SolrTestCaseJ4 {
for (int j=0; j getRanges(String id1, String id2) throws UnsupportedEncodingException {
// find minHash/maxHash hash ranges
- byte[] bytes = id1.getBytes("UTF-8");
+ byte[] bytes = id1.getBytes(StandardCharsets.UTF_8);
int minHash = Hash.murmurhash3_x86_32(bytes, 0, bytes.length, 0);
- bytes = id2.getBytes("UTF-8");
+ bytes = id2.getBytes(StandardCharsets.UTF_8);
int maxHash = Hash.murmurhash3_x86_32(bytes, 0, bytes.length, 0);
if (minHash > maxHash) {
diff --git a/solr/core/src/test/org/apache/solr/update/processor/DocExpirationUpdateProcessorFactoryTest.java b/solr/core/src/test/org/apache/solr/update/processor/DocExpirationUpdateProcessorFactoryTest.java
new file mode 100644
index 00000000000..b4e8ca0bb79
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/update/processor/DocExpirationUpdateProcessorFactoryTest.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update.processor;
+
+import org.apache.solr.common.SolrInputDocument;
+
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+import org.apache.solr.update.processor.UpdateRequestProcessorChain;
+import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
+import org.apache.solr.update.UpdateCommand;
+import org.apache.solr.update.CommitUpdateCommand;
+import org.apache.solr.update.DeleteUpdateCommand;
+
+import org.junit.BeforeClass;
+
+import java.util.Date;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Tests various configurations of DocExpirationUpdateProcessorFactory
+ */
+public class DocExpirationUpdateProcessorFactoryTest extends UpdateProcessorTestBase {
+
+ public static final String CONFIG_XML = "solrconfig-doc-expire-update-processor.xml";
+ public static final String SCHEMA_XML = "schema15.xml";
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore(CONFIG_XML, SCHEMA_XML);
+ }
+
+ public void testTTLDefaultsConversion() throws Exception {
+ SolrInputDocument d = null;
+
+ d = processAdd("convert-ttl-defaults",
+ params("NOW","1394059630042"),
+ doc(f("id", "1111"),
+ f("_ttl_","+5MINUTES")));
+ assertNotNull(d);
+ assertEquals(new Date(1394059930042L), d.getFieldValue("_expire_at_tdt"));
+
+ d = processAdd("convert-ttl-defaults",
+ params("NOW","1394059630042",
+ "_ttl_","+5MINUTES"),
+ doc(f("id", "1111")));
+
+ assertNotNull(d);
+ assertEquals(new Date(1394059930042L), d.getFieldValue("_expire_at_tdt"));
+ }
+
+ public void testTTLFieldConversion() throws Exception {
+ final String chain = "convert-ttl-field";
+ SolrInputDocument d = null;
+ d = processAdd(chain,
+ params("NOW","1394059630042"),
+ doc(f("id", "1111"),
+ f("_ttl_field_","+5MINUTES")));
+ assertNotNull(d);
+ assertEquals(new Date(1394059930042L), d.getFieldValue("_expire_at_tdt"));
+
+ d = processAdd(chain,
+ params("NOW","1394059630042"),
+ doc(f("id", "2222"),
+ f("_ttl_field_","+27MINUTES")));
+ assertNotNull(d);
+ assertEquals(new Date(1394061250042L), d.getFieldValue("_expire_at_tdt"));
+
+ d = processAdd(chain,
+ params("NOW","1394059630042"),
+ doc(f("id", "3333"),
+ f("_ttl_field_","+1YEAR")));
+ assertNotNull(d);
+ assertEquals(new Date(1425595630042L), d.getFieldValue("_expire_at_tdt"));
+
+ d = processAdd(chain,
+ params("NOW","1394059630042"),
+ doc(f("id", "1111"),
+ f("_ttl_field_","/DAY+1YEAR")));
+ assertNotNull(d);
+ assertEquals(new Date(1425513600000L), d.getFieldValue("_expire_at_tdt"));
+
+ // default ttlParamName is disabled, this should not convert...
+ d = processAdd(chain,
+ params("NOW","1394059630042",
+ "_ttl_","+5MINUTES"),
+ doc(f("id", "1111")));
+ assertNotNull(d);
+ assertNull(d.getFieldValue("_expire_at_tdt"));
+ }
+
+ public void testTTLParamConversion() throws Exception {
+ final String chain = "convert-ttl-param";
+ SolrInputDocument d = null;
+ d = processAdd(chain,
+ params("NOW","1394059630042",
+ "_ttl_param_","+5MINUTES"),
+ doc(f("id", "1111")));
+
+ assertNotNull(d);
+ assertEquals(new Date(1394059930042L), d.getFieldValue("_expire_at_tdt"));
+
+ d = processAdd(chain,
+ params("NOW","1394059630042",
+ "_ttl_param_","+27MINUTES"),
+ doc(f("id", "2222")));
+ assertNotNull(d);
+ assertEquals(new Date(1394061250042L), d.getFieldValue("_expire_at_tdt"));
+
+ // default ttlFieldName is disabled, param should be used...
+ d = processAdd(chain,
+ params("NOW","1394059630042",
+ "_ttl_param_","+5MINUTES"),
+ doc(f("id", "1111"),
+ f("_ttl_field_","+999MINUTES")));
+ assertNotNull(d);
+ assertEquals(new Date(1394059930042L), d.getFieldValue("_expire_at_tdt"));
+
+ // default ttlFieldName is disabled, this should not convert...
+ d = processAdd(chain,
+ params("NOW","1394059630042"),
+ doc(f("id", "1111"),
+ f("_ttl_","/DAY+1YEAR")));
+ assertNotNull(d);
+ assertNull(d.getFieldValue("_expire_at_tdt"));
+ }
+
+ public void testTTLFieldConversionWithDefaultParam() throws Exception {
+ final String chain = "convert-ttl-field-with-param-default";
+ SolrInputDocument d = null;
+ d = processAdd(chain,
+ params("NOW","1394059630042",
+ "_ttl_param_","+999MINUTES"),
+ doc(f("id", "1111"),
+ f("_ttl_field_","+5MINUTES")));
+ assertNotNull(d);
+ assertEquals(new Date(1394059930042L), d.getFieldValue("_expire_at_tdt"));
+
+ d = processAdd(chain,
+ params("NOW","1394059630042",
+ "_ttl_param_","+27MINUTES"),
+ doc(f("id", "2222")));
+ assertNotNull(d);
+ assertEquals(new Date(1394061250042L), d.getFieldValue("_expire_at_tdt"));
+
+ }
+
+ public void testAutomaticDeletes() throws Exception {
+
+ // get a handle on our recorder
+
+ UpdateRequestProcessorChain chain =
+ h.getCore().getUpdateProcessingChain("scheduled-delete");
+
+ assertNotNull(chain);
+
+ UpdateRequestProcessorFactory[] factories = chain.getFactories();
+ assertEquals("did number of processors configured in chain get changed?",
+ 5, factories.length);
+ assertTrue("Expected [1] RecordingUpdateProcessorFactory: " + factories[1].getClass(),
+ factories[1] instanceof RecordingUpdateProcessorFactory);
+ RecordingUpdateProcessorFactory recorder =
+ (RecordingUpdateProcessorFactory) factories[1];
+
+ // now start recording, and monitor for the expected commands
+
+ try {
+ recorder.startRecording();
+
+ // more then one iter to verify it's recurring
+ final int numItersToCheck = 1 + RANDOM_MULTIPLIER;
+
+ for (int i = 0; i < numItersToCheck; i++) {
+ UpdateCommand tmp;
+
+ // be generous in how long we wait, some jenkins machines are slooooow
+ tmp = recorder.commandQueue.poll(30, TimeUnit.SECONDS);
+
+ // we can be confident in the order because DocExpirationUpdateProcessorFactory
+ // uses the same request for both the delete & the commit -- and both
+ // RecordingUpdateProcessorFactory's getInstance & startRecording methods are
+ // synchronized. So it should not be possible to start recording in the
+ // middle of the two commands
+ assertTrue("expected DeleteUpdateCommand: " + tmp.getClass(),
+ tmp instanceof DeleteUpdateCommand);
+
+ DeleteUpdateCommand delete = (DeleteUpdateCommand) tmp;
+ assertFalse(delete.isDeleteById());
+ assertNotNull(delete.getQuery());
+ assertTrue(delete.getQuery(),
+ delete.getQuery().startsWith("{!cache=false}eXpField_tdt:[* TO "));
+
+ // commit should be immediately after the delete
+ tmp = recorder.commandQueue.poll(5, TimeUnit.SECONDS);
+ assertTrue("expected CommitUpdateCommand: " + tmp.getClass(),
+ tmp instanceof CommitUpdateCommand);
+
+ CommitUpdateCommand commit = (CommitUpdateCommand) tmp;
+ assertTrue(commit.softCommit);
+ assertTrue(commit.openSearcher);
+ }
+ } finally {
+ recorder.stopRecording();
+ }
+ }
+
+
+}
diff --git a/solr/core/src/test/org/apache/solr/update/processor/RecordingUpdateProcessorFactory.java b/solr/core/src/test/org/apache/solr/update/processor/RecordingUpdateProcessorFactory.java
new file mode 100644
index 00000000000..add57ef3257
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/update/processor/RecordingUpdateProcessorFactory.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update.processor;
+
+import java.io.IOException;
+
+import org.apache.solr.common.SolrException;
+import static org.apache.solr.common.SolrException.ErrorCode.*;
+
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.UpdateCommand;
+import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.update.CommitUpdateCommand;
+import org.apache.solr.update.DeleteUpdateCommand;
+import org.apache.solr.update.MergeIndexesCommand;
+import org.apache.solr.update.RollbackUpdateCommand;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+/**
+ * This Factory can optionally save refrences to the commands it receives in
+ * BlockingQueues that tests can poll from to observe that the exepected commands
+ * are executed. By default, this factory does nothing except return the "next"
+ * processor from the chain unless it's told to {@link #startRecording()}
+ */
+public final class RecordingUpdateProcessorFactory
+ extends UpdateRequestProcessorFactory {
+
+ private boolean recording = false;
+
+ /** The queue containing commands that were recorded
+ * @see #startRecording
+ */
+ public final BlockingQueue commandQueue
+ = new LinkedBlockingQueue();
+
+ /**
+ * @see #stopRecording
+ * @see #commandQueue
+ */
+ public synchronized void startRecording() {
+ recording = true;
+ }
+
+ /** @see #startRecording */
+ public synchronized void stopRecording() {
+ recording = false;
+ }
+
+ @Override
+ public synchronized UpdateRequestProcessor getInstance(SolrQueryRequest req,
+ SolrQueryResponse rsp,
+ UpdateRequestProcessor next ) {
+ return recording ? new RecordingUpdateRequestProcessor(commandQueue, next) : next;
+ }
+
+ private static final class RecordingUpdateRequestProcessor
+ extends UpdateRequestProcessor {
+
+ private final BlockingQueue commandQueue;
+
+ public RecordingUpdateRequestProcessor(BlockingQueue commandQueue,
+ UpdateRequestProcessor next) {
+ super(next);
+ this.commandQueue = commandQueue;
+ }
+
+ private void record(UpdateCommand cmd) {
+ if (! commandQueue.offer(cmd) ) {
+ throw new RuntimeException
+ ("WTF: commandQueue should be unbounded but offer failed: " + cmd.toString());
+ }
+ }
+
+ @Override
+ public void processAdd(AddUpdateCommand cmd) throws IOException {
+ record(cmd);
+ super.processAdd(cmd);
+ }
+ @Override
+ public void processDelete(DeleteUpdateCommand cmd) throws IOException {
+ record(cmd);
+ super.processDelete(cmd);
+ }
+ @Override
+ public void processMergeIndexes(MergeIndexesCommand cmd) throws IOException {
+ record(cmd);
+ super.processMergeIndexes(cmd);
+ }
+ @Override
+ public void processCommit(CommitUpdateCommand cmd) throws IOException {
+ record(cmd);
+ super.processCommit(cmd);
+ }
+ @Override
+ public void processRollback(RollbackUpdateCommand cmd) throws IOException {
+ record(cmd);
+ super.processRollback(cmd);
+ }
+ }
+}
+
+
+
diff --git a/solr/core/src/test/org/apache/solr/update/processor/UpdateProcessorTestBase.java b/solr/core/src/test/org/apache/solr/update/processor/UpdateProcessorTestBase.java
index 5ddff4555d0..8d849d7be9c 100644
--- a/solr/core/src/test/org/apache/solr/update/processor/UpdateProcessorTestBase.java
+++ b/solr/core/src/test/org/apache/solr/update/processor/UpdateProcessorTestBase.java
@@ -24,6 +24,7 @@ import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
@@ -65,14 +66,19 @@ public class UpdateProcessorTestBase extends SolrTestCaseJ4 {
SolrQueryRequest req = new LocalSolrQueryRequest(core, requestParams);
try {
+ SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
AddUpdateCommand cmd = new AddUpdateCommand(req);
cmd.solrDoc = docIn;
UpdateRequestProcessor processor = pc.createProcessor(req, rsp);
- processor.processAdd(cmd);
+ if (null != processor) {
+ // test chain might be empty or short circuted.
+ processor.processAdd(cmd);
+ }
return cmd.solrDoc;
} finally {
+ SolrRequestInfo.clearRequestInfo();
req.close();
}
}
diff --git a/solr/core/src/test/org/apache/solr/util/SimplePostToolTest.java b/solr/core/src/test/org/apache/solr/util/SimplePostToolTest.java
index 8655bd95c02..2b93cebd107 100644
--- a/solr/core/src/test/org/apache/solr/util/SimplePostToolTest.java
+++ b/solr/core/src/test/org/apache/solr/util/SimplePostToolTest.java
@@ -24,6 +24,7 @@ import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Set;
@@ -213,7 +214,7 @@ public class SimplePostToolTest extends SolrTestCaseJ4 {
sb.append("Disallow: /disallow # Disallow this path\n");
sb.append("Disallow: /nonexistingpath # Disallow this path\n");
this.robotsCache.put("[ff01::114]", SimplePostTool.pageFetcher.
- parseRobotsTxt(new ByteArrayInputStream(sb.toString().getBytes("UTF-8"))));
+ parseRobotsTxt(new ByteArrayInputStream(sb.toString().getBytes(StandardCharsets.UTF_8))));
}
@Override
@@ -225,11 +226,7 @@ public class SimplePostToolTest extends SolrTestCaseJ4 {
}
res.httpStatus = 200;
res.contentType = "text/html";
- try {
- res.content = htmlMap.get(u.toString()).getBytes("UTF-8");
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException();
- }
+ res.content = htmlMap.get(u.toString()).getBytes(StandardCharsets.UTF_8);
return res;
}
diff --git a/solr/example/solr/collection1/conf/_schema_analysis_synonyms_english.json b/solr/example/solr/collection1/conf/_schema_analysis_synonyms_english.json
new file mode 100644
index 00000000000..869bdce0514
--- /dev/null
+++ b/solr/example/solr/collection1/conf/_schema_analysis_synonyms_english.json
@@ -0,0 +1,11 @@
+{
+ "initArgs":{
+ "ignoreCase":true,
+ "format":"solr"
+ },
+ "managedMap":{
+ "GB":["GiB","Gigabyte"],
+ "happy":["glad","joyful"],
+ "TV":["Television"]
+ }
+}
diff --git a/solr/example/solr/collection1/conf/schema.xml b/solr/example/solr/collection1/conf/schema.xml
index adaedfdbf8a..5504a0f00f1 100755
--- a/solr/example/solr/collection1/conf/schema.xml
+++ b/solr/example/solr/collection1/conf/schema.xml
@@ -453,6 +453,7 @@
diff --git a/solr/licenses/junit4-ant-2.1.1.jar.sha1 b/solr/licenses/junit4-ant-2.1.1.jar.sha1
deleted file mode 100644
index 4340e4c8609..00000000000
--- a/solr/licenses/junit4-ant-2.1.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a8a7371e11a8b3a4a3eeea81ad3cedafe3e3550e
diff --git a/solr/licenses/junit4-ant-2.1.3.jar.sha1 b/solr/licenses/junit4-ant-2.1.3.jar.sha1
new file mode 100644
index 00000000000..c2d6fa49fa8
--- /dev/null
+++ b/solr/licenses/junit4-ant-2.1.3.jar.sha1
@@ -0,0 +1 @@
+8636804644d4ae3874f0efaa98978887e171cd55
diff --git a/solr/licenses/randomizedtesting-runner-2.1.1.jar.sha1 b/solr/licenses/randomizedtesting-runner-2.1.1.jar.sha1
deleted file mode 100644
index 2923eedf9fe..00000000000
--- a/solr/licenses/randomizedtesting-runner-2.1.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5908c4e714dab40ccc892993a21537c7c0d6210c
diff --git a/solr/licenses/randomizedtesting-runner-2.1.3.jar.sha1 b/solr/licenses/randomizedtesting-runner-2.1.3.jar.sha1
new file mode 100644
index 00000000000..5da2ec2946a
--- /dev/null
+++ b/solr/licenses/randomizedtesting-runner-2.1.3.jar.sha1
@@ -0,0 +1 @@
+d340caee99857ed0384681eea6219a4d937e7ee4
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java
index dadc235773e..5a9af1e89d8 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java
@@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.impl;
import java.io.IOException;
import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
import java.util.LinkedList;
import java.util.Locale;
import java.util.Queue;
@@ -172,7 +173,7 @@ public class ConcurrentUpdateSolrServer extends SolrServer {
public void writeTo(OutputStream out) throws IOException {
try {
if (isXml) {
- out.write("".getBytes("UTF-8")); // can be anything
+ out.write("".getBytes(StandardCharsets.UTF_8)); // can be anything
}
UpdateRequest req = updateRequest;
while (req != null) {
@@ -197,7 +198,7 @@ public class ConcurrentUpdateSolrServer extends SolrServer {
byte[] content = String.format(Locale.ROOT,
fmt,
params.getBool(UpdateParams.WAIT_SEARCHER, false)
- + "").getBytes("UTF-8");
+ + "").getBytes(StandardCharsets.UTF_8);
out.write(content);
}
}
@@ -206,7 +207,7 @@ public class ConcurrentUpdateSolrServer extends SolrServer {
req = queue.poll(pollQueueTime, TimeUnit.MILLISECONDS);
}
if (isXml) {
- out.write(" ".getBytes("UTF-8"));
+ out.write(" ".getBytes(StandardCharsets.UTF_8));
}
} catch (InterruptedException e) {
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java
index 040b20c829a..92b5ac22f50 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java
@@ -21,6 +21,7 @@ import java.io.InputStream;
import java.net.ConnectException;
import java.net.SocketTimeoutException;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
@@ -78,7 +79,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class HttpSolrServer extends SolrServer {
- private static final String UTF_8 = "UTF-8";
+ private static final String UTF_8 = StandardCharsets.UTF_8.name();
private static final String DEFAULT_PATH = "/select";
private static final long serialVersionUID = -946812319974801896L;
@@ -335,7 +336,7 @@ public class HttpSolrServer extends SolrServer {
if (vals != null) {
for (String v : vals) {
if (isMultipart) {
- parts.add(new FormBodyPart(p, new StringBody(v, Charset.forName("UTF-8"))));
+ parts.add(new FormBodyPart(p, new StringBody(v, StandardCharsets.UTF_8)));
} else {
postParams.add(new BasicNameValuePair(p, v));
}
@@ -369,7 +370,7 @@ public class HttpSolrServer extends SolrServer {
post.setEntity(entity);
} else {
//not using multipart
- post.setEntity(new UrlEncodedFormEntity(postParams, "UTF-8"));
+ post.setEntity(new UrlEncodedFormEntity(postParams, StandardCharsets.UTF_8));
}
method = post;
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java
index a1ebefe9ddd..858795719bf 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java
@@ -298,85 +298,17 @@ public class LBHttpSolrServer extends SolrServer {
rsp.server = serverStr;
HttpSolrServer server = makeServer(serverStr);
- try {
- rsp.rsp = server.request(req.getRequest());
+ ex = doRequest(server, req, rsp, isUpdate, false, null);
+ if (ex == null) {
return rsp; // SUCCESS
- } catch (SolrException e) {
- // we retry on 404 or 403 or 503 or 500
- // unless it's an update - then we only retry on connect exceptions
- if (!isUpdate && RETRY_CODES.contains(e.code())) {
- ex = addZombie(server, e);
- } else {
- // Server is alive but the request was likely malformed or invalid
- throw e;
- }
- } catch (SocketException e) {
- if (!isUpdate || e instanceof ConnectException) {
- ex = addZombie(server, e);
- } else {
- throw e;
- }
- } catch (SocketTimeoutException e) {
- if (!isUpdate) {
- ex = addZombie(server, e);
- } else {
- throw e;
- }
- } catch (SolrServerException e) {
- Throwable rootCause = e.getRootCause();
- if (!isUpdate && rootCause instanceof IOException) {
- ex = addZombie(server, e);
- } else if (isUpdate && rootCause instanceof ConnectException) {
- ex = addZombie(server, e);
- } else {
- throw e;
- }
- } catch (Exception e) {
- throw new SolrServerException(e);
}
}
// try the servers we previously skipped
for (ServerWrapper wrapper : skipped) {
- try {
- rsp.rsp = wrapper.solrServer.request(req.getRequest());
- zombieServers.remove(wrapper.getKey());
- return rsp; // SUCCESS
- } catch (SolrException e) {
- // we retry on 404 or 403 or 503 or 500
- // unless it's an update - then we only retry on connect exceptions
- if (!isUpdate && RETRY_CODES.contains(e.code())) {
- ex = e;
- // already a zombie, no need to re-add
- } else {
- // Server is alive but the request was malformed or invalid
- zombieServers.remove(wrapper.getKey());
- throw e;
- }
-
- } catch (SocketException e) {
- if (!isUpdate || e instanceof ConnectException) {
- ex = e;
- } else {
- throw e;
- }
- } catch (SocketTimeoutException e) {
- if (!isUpdate) {
- ex = e;
- } else {
- throw e;
- }
- } catch (SolrServerException e) {
- Throwable rootCause = e.getRootCause();
- if (!isUpdate && rootCause instanceof IOException) {
- ex = e;
- } else if (isUpdate && rootCause instanceof ConnectException) {
- ex = e;
- } else {
- throw e;
- }
- } catch (Exception e) {
- throw new SolrServerException(e);
+ ex = doRequest(wrapper.solrServer, req, rsp, isUpdate, true, wrapper.getKey());
+ if (ex == null) {
+ return rsp; // SUCCESS
}
}
@@ -401,7 +333,53 @@ public class LBHttpSolrServer extends SolrServer {
return e;
}
+ protected Exception doRequest(HttpSolrServer server, Req req, Rsp rsp, boolean isUpdate,
+ boolean isZombie, String zombieKey) throws SolrServerException, IOException {
+ Exception ex = null;
+ try {
+ rsp.rsp = server.request(req.getRequest());
+ if (isZombie) {
+ zombieServers.remove(zombieKey);
+ }
+ } catch (SolrException e) {
+ // we retry on 404 or 403 or 503 or 500
+ // unless it's an update - then we only retry on connect exception
+ if (!isUpdate && RETRY_CODES.contains(e.code())) {
+ ex = (!isZombie) ? addZombie(server, e) : e;
+ } else {
+ // Server is alive but the request was likely malformed or invalid
+ if (isZombie) {
+ zombieServers.remove(zombieKey);
+ }
+ throw e;
+ }
+ } catch (SocketException e) {
+ if (!isUpdate || e instanceof ConnectException) {
+ ex = (!isZombie) ? addZombie(server, e) : e;
+ } else {
+ throw e;
+ }
+ } catch (SocketTimeoutException e) {
+ if (!isUpdate) {
+ ex = (!isZombie) ? addZombie(server, e) : e;
+ } else {
+ throw e;
+ }
+ } catch (SolrServerException e) {
+ Throwable rootCause = e.getRootCause();
+ if (!isUpdate && rootCause instanceof IOException) {
+ ex = (!isZombie) ? addZombie(server, e) : e;
+ } else if (isUpdate && rootCause instanceof ConnectException) {
+ ex = (!isZombie) ? addZombie(server, e) : e;
+ } else {
+ throw e;
+ }
+ } catch (Exception e) {
+ throw new SolrServerException(e);
+ }
+ return ex;
+ }
private void updateAliveList() {
synchronized (aliveServers) {
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/AbstractUpdateRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/AbstractUpdateRequest.java
index dce10455290..acfc525b33e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/AbstractUpdateRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/AbstractUpdateRequest.java
@@ -83,6 +83,12 @@ public abstract class AbstractUpdateRequest extends SolrRequest implements IsUpd
return setAction(action, waitFlush, waitSearcher,maxSegments,false,expungeDeletes);
}
+ public AbstractUpdateRequest setAction(ACTION action, boolean waitFlush, boolean waitSearcher, int maxSegments, boolean softCommit, boolean expungeDeletes, boolean openSearcher) {
+ setAction(action, waitFlush, waitSearcher, maxSegments, softCommit, expungeDeletes);
+ params.set(UpdateParams.OPEN_SEARCHER, String.valueOf(openSearcher));
+ return this;
+ }
+
/**
* @since Solr 1.4
*/
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/RequestWriter.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/RequestWriter.java
index 2990653c6ea..a33fe7e377a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/RequestWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/RequestWriter.java
@@ -28,6 +28,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
/**
* A RequestWriter is used to write requests to Solr.
@@ -38,7 +39,7 @@ import java.nio.charset.Charset;
* @since solr 1.4
*/
public class RequestWriter {
- public static final Charset UTF_8 = Charset.forName("UTF-8");
+ public static final Charset UTF_8 = StandardCharsets.UTF_8;
public Collection getContentStreams(SolrRequest req) throws IOException {
if (req instanceof UpdateRequest) {
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
index a2873e0c9c4..d8ea26859ed 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.StringReader;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
@@ -505,22 +506,17 @@ public class SolrZkClient {
}
string.append(dent + path + " (" + children.size() + ")" + NEWL);
if (data != null) {
- try {
- String dataString = new String(data, "UTF-8");
- if ((!path.endsWith(".txt") && !path.endsWith(".xml")) || path.endsWith(ZkStateReader.CLUSTER_STATE)) {
- if (path.endsWith(".xml")) {
- // this is the cluster state in xml format - lets pretty print
- dataString = prettyPrint(dataString);
- }
-
- string.append(dent + "DATA:\n" + dent + " "
- + dataString.replaceAll("\n", "\n" + dent + " ") + NEWL);
- } else {
- string.append(dent + "DATA: ...supressed..." + NEWL);
+ String dataString = new String(data, StandardCharsets.UTF_8);
+ if ((!path.endsWith(".txt") && !path.endsWith(".xml")) || path.endsWith(ZkStateReader.CLUSTER_STATE)) {
+ if (path.endsWith(".xml")) {
+ // this is the cluster state in xml format - lets pretty print
+ dataString = prettyPrint(dataString);
}
- } catch (UnsupportedEncodingException e) {
- // can't happen - UTF-8
- throw new RuntimeException(e);
+
+ string.append(dent + "DATA:\n" + dent + " "
+ + dataString.replaceAll("\n", "\n" + dent + " ") + NEWL);
+ } else {
+ string.append(dent + "DATA: ...supressed..." + NEWL);
}
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
index 71fb24ceda9..d8a50b35576 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
@@ -44,7 +44,9 @@ public interface CollectionParams
CLUSTERPROP,
REQUESTSTATUS,
ADDREPLICA,
- OVERSEERSTATUS;
+ OVERSEERSTATUS,
+ LIST,
+ CLUSTERSTATUS;
public static CollectionAction get( String p )
{
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/ExpandParams.java b/solr/solrj/src/java/org/apache/solr/common/params/ExpandParams.java
index 55f37335c15..a8f0cf7dc03 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/ExpandParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/ExpandParams.java
@@ -25,7 +25,8 @@ public interface ExpandParams {
public static final String EXPAND = "expand";
public static final String EXPAND_SORT = EXPAND + ".sort";
public static final String EXPAND_ROWS = EXPAND + ".rows";
-
-
+ public static final String EXPAND_FIELD = EXPAND + ".field";
+ public static final String EXPAND_Q = EXPAND + ".q";
+ public static final String EXPAND_FQ = EXPAND + ".fq";
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java b/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java
index d531b0faf33..34238b5f928 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/ContentStreamBase.java
@@ -27,6 +27,7 @@ import java.io.Reader;
import java.io.StringReader;
import java.net.URL;
import java.net.URLConnection;
+import java.nio.charset.StandardCharsets;
import java.util.Locale;
/**
@@ -37,7 +38,7 @@ import java.util.Locale;
*/
public abstract class ContentStreamBase implements ContentStream
{
- public static final String DEFAULT_CHARSET = "utf-8";
+ public static final String DEFAULT_CHARSET = StandardCharsets.UTF_8.name();
protected String name;
protected String sourceInfo;
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java
index 3d469b0d672..3e45fa06ace 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrServerTest.java
@@ -238,9 +238,9 @@ public class CloudSolrServerTest extends AbstractFullDistribZkTestBase {
// Calculate a number of shard keys that route to the same shard.
int n;
if (TEST_NIGHTLY) {
- n = random().nextInt(999) + 1;
+ n = random().nextInt(999) + 2;
} else {
- n = random().nextInt(9) + 1;
+ n = random().nextInt(9) + 2;
}
List sameShardRoutes = Lists.newArrayList();
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/response/NoOpResponseParserTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/response/NoOpResponseParserTest.java
index 490d2f0870f..448d295358a 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/response/NoOpResponseParserTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/response/NoOpResponseParserTest.java
@@ -41,6 +41,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
+import java.nio.charset.StandardCharsets;
import java.util.List;
/**
@@ -102,7 +103,7 @@ public class NoOpResponseParserTest extends SolrJettyTestBase {
NoOpResponseParser parser = new NoOpResponseParser();
try (final InputStream is = getResponse()) {
assertNotNull(is);
- Reader in = new InputStreamReader(is, "UTF-8");
+ Reader in = new InputStreamReader(is, StandardCharsets.UTF_8);
NamedList response = parser.processResponse(in);
assertNotNull(response.get("response"));
String expectedResponse = IOUtils.toString(getResponse(), "UTF-8");
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/response/QueryResponseTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/response/QueryResponseTest.java
index 1c1b70a9e90..c5692fe1007 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/response/QueryResponseTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/response/QueryResponseTest.java
@@ -18,6 +18,7 @@
package org.apache.solr.client.solrj.response;
import junit.framework.Assert;
+
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.impl.XMLResponseParser;
import org.apache.solr.common.SolrDocumentList;
@@ -29,6 +30,7 @@ import org.junit.Test;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
+import java.nio.charset.StandardCharsets;
import java.util.List;
/**
@@ -42,7 +44,7 @@ public class QueryResponseTest extends LuceneTestCase {
XMLResponseParser parser = new XMLResponseParser();
InputStream is = new SolrResourceLoader(null, null).openResource("solrj/sampleDateFacetResponse.xml");
assertNotNull(is);
- Reader in = new InputStreamReader(is, "UTF-8");
+ Reader in = new InputStreamReader(is, StandardCharsets.UTF_8);
NamedList response = parser.processResponse(in);
in.close();
@@ -66,7 +68,7 @@ public class QueryResponseTest extends LuceneTestCase {
XMLResponseParser parser = new XMLResponseParser();
InputStream is = new SolrResourceLoader(null, null).openResource("solrj/sampleDateFacetResponse.xml");
assertNotNull(is);
- Reader in = new InputStreamReader(is, "UTF-8");
+ Reader in = new InputStreamReader(is, StandardCharsets.UTF_8);
NamedList response = parser.processResponse(in);
in.close();
@@ -123,7 +125,7 @@ public class QueryResponseTest extends LuceneTestCase {
XMLResponseParser parser = new XMLResponseParser();
InputStream is = new SolrResourceLoader(null, null).openResource("solrj/sampleGroupResponse.xml");
assertNotNull(is);
- Reader in = new InputStreamReader(is, "UTF-8");
+ Reader in = new InputStreamReader(is, StandardCharsets.UTF_8);
NamedList response = parser.processResponse(in);
in.close();
@@ -225,7 +227,7 @@ public class QueryResponseTest extends LuceneTestCase {
XMLResponseParser parser = new XMLResponseParser();
InputStream is = new SolrResourceLoader(null, null).openResource("solrj/sampleSimpleGroupResponse.xml");
assertNotNull(is);
- Reader in = new InputStreamReader(is, "UTF-8");
+ Reader in = new InputStreamReader(is, StandardCharsets.UTF_8);
NamedList response = parser.processResponse(in);
in.close();
diff --git a/solr/solrj/src/test/org/apache/solr/common/util/ContentStreamTest.java b/solr/solrj/src/test/org/apache/solr/common/util/ContentStreamTest.java
index 22c3d85fdb0..2945a189cb6 100644
--- a/solr/solrj/src/test/org/apache/solr/common/util/ContentStreamTest.java
+++ b/solr/solrj/src/test/org/apache/solr/common/util/ContentStreamTest.java
@@ -25,6 +25,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.io.IOUtils;
import org.apache.solr.SolrTestCaseJ4;
@@ -58,13 +59,15 @@ public class ContentStreamTest extends SolrTestCaseJ4
InputStream s = stream.getStream();
FileInputStream fis = new FileInputStream(file);
InputStreamReader isr = new InputStreamReader(
- new FileInputStream(file), "UTF-8");
+ new FileInputStream(file), StandardCharsets.UTF_8);
+ Reader r = stream.getReader();
try {
assertEquals(file.length(), stream.getSize().intValue());
assertTrue(IOUtils.contentEquals(fis, s));
- assertTrue(IOUtils.contentEquals(isr, stream.getReader()));
+ assertTrue(IOUtils.contentEquals(isr, r));
} finally {
s.close();
+ r.close();
isr.close();
fis.close();
}
@@ -86,7 +89,7 @@ public class ContentStreamTest extends SolrTestCaseJ4
InputStream s = stream.getStream();
FileInputStream fis = new FileInputStream(file);
FileInputStream fis2 = new FileInputStream(file);
- InputStreamReader isr = new InputStreamReader(fis, "UTF-8");
+ InputStreamReader isr = new InputStreamReader(fis, StandardCharsets.UTF_8);
Reader r = stream.getReader();
try {
assertTrue(IOUtils.contentEquals(fis2, s));
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
index 1aeb7c572b1..fe13e345703 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
@@ -325,7 +325,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
if (xmlStr == null) {
xmlStr = " onlyInB = new HashSet<>(setB);
onlyInB.removeAll(setA);
- if (onlyInA.size() > 0) {
- for (SolrDocument doc : onlyInA) {
- if (!addFails.contains(doc.getFirstValue("id"))) {
- legal = false;
- } else {
- System.err.println("###### Only in " + aName + ": " + onlyInA
- + ", but this is expected because we found an add fail for "
- + doc.getFirstValue("id"));
- }
+ for (SolrDocument doc : onlyInA) {
+ if (!addFails.contains(doc.getFirstValue("id"))) {
+ legal = false;
+ } else {
+ System.err.println("###### Only in " + aName + ": " + onlyInA
+ + ", but this is expected because we found an add fail for "
+ + doc.getFirstValue("id"));
}
-
}
- if (onlyInB.size() > 0) {
- for (SolrDocument doc : onlyInB) {
- if (!deleteFails.contains(doc.getFirstValue("id"))) {
- legal = false;
- } else {
- System.err.println("###### Only in " + bName + ": " + onlyInB
- + ", but this is expected because we found a delete fail for "
- + doc.getFirstValue("id"));
- }
+
+ for (SolrDocument doc : onlyInB) {
+ if (!deleteFails.contains(doc.getFirstValue("id"))) {
+ legal = false;
+ } else {
+ System.err.println("###### Only in " + bName + ": " + onlyInB
+ + ", but this is expected because we found a delete fail for "
+ + doc.getFirstValue("id"));
}
}
@@ -1654,8 +1650,12 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
if (client == null) {
final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(clientIndex));
SolrServer server = createNewSolrServer("", baseUrl);
- res.setResponse(server.request(request));
- server.shutdown();
+ try {
+ res.setResponse(server.request(request));
+ server.shutdown();
+ } finally {
+ if (server != null) server.shutdown();
+ }
} else {
res.setResponse(client.request(request));
}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java b/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java
index c3cc399dd87..9a044a24477 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java
@@ -26,6 +26,7 @@ import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.UnknownHostException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
@@ -314,7 +315,7 @@ public class ZkTestServer {
BufferedReader reader = null;
try {
OutputStream outstream = sock.getOutputStream();
- outstream.write(cmd.getBytes("US-ASCII"));
+ outstream.write(cmd.getBytes(StandardCharsets.US_ASCII));
outstream.flush();
// this replicates NC - close the output stream before reading
sock.shutdownOutput();
diff --git a/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java b/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java
index 94dba92736d..2b15abb4bb9 100644
--- a/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java
+++ b/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java
@@ -28,10 +28,12 @@ import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
+
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
abstract public class BaseTestHarness {
private static final ThreadLocal builderTL = new ThreadLocal<>();
@@ -80,7 +82,7 @@ abstract public class BaseTestHarness {
Document document = null;
try {
document = getXmlDocumentBuilder().parse(new ByteArrayInputStream
- (xml.getBytes("UTF-8")));
+ (xml.getBytes(StandardCharsets.UTF_8)));
} catch (UnsupportedEncodingException e1) {
throw new RuntimeException("Totally weird UTF-8 exception", e1);
} catch (IOException e2) {
@@ -105,7 +107,7 @@ abstract public class BaseTestHarness {
Document document = null;
try {
document = getXmlDocumentBuilder().parse(new ByteArrayInputStream
- (xml.getBytes("UTF-8")));
+ (xml.getBytes(StandardCharsets.UTF_8)));
} catch (UnsupportedEncodingException e1) {
throw new RuntimeException("Totally weird UTF-8 exception", e1);
} catch (IOException e2) {
diff --git a/solr/test-framework/src/java/org/apache/solr/util/RestTestHarness.java b/solr/test-framework/src/java/org/apache/solr/util/RestTestHarness.java
index e298947fd24..0935d6ed5ec 100644
--- a/solr/test-framework/src/java/org/apache/solr/util/RestTestHarness.java
+++ b/solr/test-framework/src/java/org/apache/solr/util/RestTestHarness.java
@@ -18,6 +18,7 @@ package org.apache.solr.util;
import java.io.IOException;
import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
@@ -106,7 +107,7 @@ public class RestTestHarness extends BaseTestHarness {
public String put(String request, String content) throws IOException {
HttpPut httpPut = new HttpPut(getBaseURL() + request);
httpPut.setEntity(new StringEntity(content, ContentType.create(
- "application/json", "utf-8")));
+ "application/json", StandardCharsets.UTF_8)));
return getResponse(httpPut);
}
@@ -134,7 +135,7 @@ public class RestTestHarness extends BaseTestHarness {
public String post(String request, String content) throws IOException {
HttpPost httpPost = new HttpPost(getBaseURL() + request);
httpPost.setEntity(new StringEntity(content, ContentType.create(
- "application/json", "utf-8")));
+ "application/json", StandardCharsets.UTF_8)));
return getResponse(httpPost);
}
@@ -189,7 +190,7 @@ public class RestTestHarness extends BaseTestHarness {
HttpEntity entity = null;
try {
entity = httpClient.execute(request).getEntity();
- return EntityUtils.toString(entity, "UTF-8");
+ return EntityUtils.toString(entity, StandardCharsets.UTF_8);
} finally {
EntityUtils.consumeQuietly(entity);
}