mirror of https://github.com/apache/lucene.git
Merge remote-tracking branch 'origin/master' into gradle-master
This commit is contained in:
Dawid Weiss
commit
f9dde4de52
|
|
@ -60,7 +60,7 @@ CharacterEntities = ( "AElig" | "Aacute" | "Acirc" | "Agrave" | "Alpha"
|
|||
| "times" | "trade" | "uArr" | "uacute" | "uarr" | "ucirc"
|
||||
| "ugrave" | "uml" | "upsih" | "upsilon" | "uuml"
|
||||
| "weierp" | "xi" | "yacute" | "yen" | "yuml" | "zeta"
|
||||
| "zwj" | "zwnj" )
|
||||
(' | "zwj" | "zwnj"', ')')
|
||||
%{
|
||||
private static final Map<String,String> upperCaseVariantsAccepted
|
||||
= new HashMap<>();
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -24,7 +24,7 @@ import org.apache.lucene.util.SparseFixedBitSet;
|
|||
|
||||
/**
|
||||
* This file contains unicode properties used by various {@link CharTokenizer}s.
|
||||
* The data was created using ICU4J v62.1.0.0
|
||||
* The data was created using ICU4J v62.2.0.0
|
||||
* <p>
|
||||
* Unicode version: 11.0.0.0
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -61,9 +61,8 @@ import java.util.regex.Pattern;
|
|||
* </ol>
|
||||
*/
|
||||
public class GenerateUTR30DataFiles {
|
||||
private static final String ICU_SVN_TAG_URL
|
||||
= "http://source.icu-project.org/repos/icu/tags";
|
||||
private static final String ICU_RELEASE_TAG = "release-62-1";
|
||||
private static final String ICU_GIT_TAG_URL = "https://raw.githubusercontent.com/unicode-org/icu";
|
||||
private static final String ICU_RELEASE_TAG = "maint/maint-62";
|
||||
private static final String ICU_DATA_NORM2_PATH = "icu4c/source/data/unidata/norm2";
|
||||
private static final String NFC_TXT = "nfc.txt";
|
||||
private static final String NFKC_TXT = "nfkc.txt";
|
||||
|
|
@ -165,7 +164,7 @@ public class GenerateUTR30DataFiles {
|
|||
}
|
||||
|
||||
private static void getNFKCDataFilesFromIcuProject() throws IOException {
|
||||
URL icuTagsURL = new URL(ICU_SVN_TAG_URL + "/");
|
||||
URL icuTagsURL = new URL(ICU_GIT_TAG_URL + "/");
|
||||
URL icuReleaseTagURL = new URL(icuTagsURL, ICU_RELEASE_TAG + "/");
|
||||
URL norm2url = new URL(icuReleaseTagURL, ICU_DATA_NORM2_PATH + "/");
|
||||
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -95,25 +95,13 @@
|
|||
<arg value="-B"/>
|
||||
<arg value="gen_BulkOperation.py"/>
|
||||
</exec>
|
||||
<exec dir="src/java/org/apache/lucene/util/packed"
|
||||
executable="${python.exe}" failonerror="true">
|
||||
<!-- Tell Python not to write any bytecode cache into the filesystem: -->
|
||||
<arg value="-B"/>
|
||||
<arg value="gen_Direct.py"/>
|
||||
</exec>
|
||||
<exec dir="src/java/org/apache/lucene/util/packed"
|
||||
executable="${python.exe}" failonerror="true">
|
||||
<!-- Tell Python not to write any bytecode cache into the filesystem: -->
|
||||
<arg value="-B"/>
|
||||
<arg value="gen_Packed64SingleBlock.py"/>
|
||||
</exec>
|
||||
<exec dir="src/java/org/apache/lucene/util/packed"
|
||||
executable="${python.exe}" failonerror="true">
|
||||
<!-- Tell Python not to write any bytecode cache into the filesystem: -->
|
||||
<arg value="-B"/>
|
||||
<arg value="gen_PackedThreeBlocks.py"/>
|
||||
</exec>
|
||||
<fixcrlf srcdir="src/java/org/apache/lucene/util/packed" includes="BulkOperation*.java,Direct*.java,Packed64SingleBlock.java,Packed*ThreeBlocks.py" encoding="UTF-8"/>
|
||||
<fixcrlf srcdir="src/java/org/apache/lucene/util/packed" includes="BulkOperation*.java,Packed64SingleBlock.java" encoding="UTF-8"/>
|
||||
</target>
|
||||
|
||||
<target name="createLevAutomata" depends="check-moman,download-moman">
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ abstract class Packed64SingleBlock extends PackedInts.MutableImpl {
|
|||
public long ramBytesUsed() {
|
||||
return RamUsageEstimator.alignObjectSize(
|
||||
RamUsageEstimator.NUM_BYTES_OBJECT_HEADER
|
||||
+ 2 * Integer.BYTES // valueCount,bitsPerValue
|
||||
+ 2 * Integer.BYTES // valueCount,bitsPerValue
|
||||
+ RamUsageEstimator.NUM_BYTES_OBJECT_REF) // blocks ref
|
||||
+ RamUsageEstimator.sizeOf(blocks);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,175 +0,0 @@
|
|||
#! /usr/bin/env python
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
HEADER = """// This file has been automatically generated, DO NOT EDIT
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.lucene.util.packed;
|
||||
|
||||
import org.apache.lucene.store.DataInput;
|
||||
import org.apache.lucene.util.RamUsageEstimator;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
|
||||
"""
|
||||
|
||||
TYPES = {8: "byte", 16: "short", 32: "int", 64: "long"}
|
||||
MASKS = {8: " & 0xFFL", 16: " & 0xFFFFL", 32: " & 0xFFFFFFFFL", 64: ""}
|
||||
CASTS = {8: "(byte) ", 16: "(short) ", 32: "(int) ", 64: ""}
|
||||
|
||||
if __name__ == '__main__':
|
||||
for bpv in TYPES.keys():
|
||||
type
|
||||
f = open("Direct%d.java" % bpv, 'w')
|
||||
f.write(HEADER)
|
||||
f.write("""/**
|
||||
* Direct wrapping of %d-bits values to a backing array.
|
||||
* @lucene.internal
|
||||
*/\n""" % bpv)
|
||||
f.write("final class Direct%d extends PackedInts.MutableImpl {\n" % bpv)
|
||||
f.write(" final %s[] values;\n\n" % TYPES[bpv])
|
||||
|
||||
f.write(" Direct%d(int valueCount) {\n" % bpv)
|
||||
f.write(" super(valueCount, %d);\n" % bpv)
|
||||
f.write(" values = new %s[valueCount];\n" % TYPES[bpv])
|
||||
f.write(" }\n\n")
|
||||
|
||||
f.write(" Direct%d(int packedIntsVersion, DataInput in, int valueCount) throws IOException {\n" % bpv)
|
||||
f.write(" this(valueCount);\n")
|
||||
if bpv == 8:
|
||||
f.write(" in.readBytes(values, 0, valueCount);\n")
|
||||
else:
|
||||
f.write(" for (int i = 0; i < valueCount; ++i) {\n")
|
||||
f.write(" values[i] = in.read%s();\n" % TYPES[bpv].title())
|
||||
f.write(" }\n")
|
||||
if bpv != 64:
|
||||
f.write(" // because packed ints have not always been byte-aligned\n")
|
||||
f.write(" final int remaining = (int) (PackedInts.Format.PACKED.byteCount(packedIntsVersion, valueCount, %d) - %dL * valueCount);\n" % (bpv, bpv / 8))
|
||||
f.write(" for (int i = 0; i < remaining; ++i) {\n")
|
||||
f.write(" in.readByte();\n")
|
||||
f.write(" }\n")
|
||||
f.write(" }\n")
|
||||
|
||||
f.write("""
|
||||
@Override
|
||||
public long get(final int index) {
|
||||
return values[index]%s;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void set(final int index, final long value) {
|
||||
values[index] = %s(value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long ramBytesUsed() {
|
||||
return RamUsageEstimator.alignObjectSize(
|
||||
RamUsageEstimator.NUM_BYTES_OBJECT_HEADER
|
||||
+ 2 * RamUsageEstimator.NUM_BYTES_INT // valueCount,bitsPerValue
|
||||
+ RamUsageEstimator.NUM_BYTES_OBJECT_REF) // values ref
|
||||
+ RamUsageEstimator.sizeOf(values);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clear() {
|
||||
Arrays.fill(values, %s0L);
|
||||
}
|
||||
""" % (MASKS[bpv], CASTS[bpv], CASTS[bpv]))
|
||||
|
||||
if bpv == 64:
|
||||
f.write("""
|
||||
@Override
|
||||
public int get(int index, long[] arr, int off, int len) {
|
||||
assert len > 0 : "len must be > 0 (got " + len + ")";
|
||||
assert index >= 0 && index < valueCount;
|
||||
assert off + len <= arr.length;
|
||||
|
||||
final int gets = Math.min(valueCount - index, len);
|
||||
System.arraycopy(values, index, arr, off, gets);
|
||||
return gets;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int set(int index, long[] arr, int off, int len) {
|
||||
assert len > 0 : "len must be > 0 (got " + len + ")";
|
||||
assert index >= 0 && index < valueCount;
|
||||
assert off + len <= arr.length;
|
||||
|
||||
final int sets = Math.min(valueCount - index, len);
|
||||
System.arraycopy(arr, off, values, index, sets);
|
||||
return sets;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fill(int fromIndex, int toIndex, long val) {
|
||||
Arrays.fill(values, fromIndex, toIndex, val);
|
||||
}
|
||||
""")
|
||||
else:
|
||||
f.write("""
|
||||
@Override
|
||||
public int get(int index, long[] arr, int off, int len) {
|
||||
assert len > 0 : "len must be > 0 (got " + len + ")";
|
||||
assert index >= 0 && index < valueCount;
|
||||
assert off + len <= arr.length;
|
||||
|
||||
final int gets = Math.min(valueCount - index, len);
|
||||
for (int i = index, o = off, end = index + gets; i < end; ++i, ++o) {
|
||||
arr[o] = values[i]%s;
|
||||
}
|
||||
return gets;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int set(int index, long[] arr, int off, int len) {
|
||||
assert len > 0 : "len must be > 0 (got " + len + ")";
|
||||
assert index >= 0 && index < valueCount;
|
||||
assert off + len <= arr.length;
|
||||
|
||||
final int sets = Math.min(valueCount - index, len);
|
||||
for (int i = index, o = off, end = index + sets; i < end; ++i, ++o) {
|
||||
values[i] = %sarr[o];
|
||||
}
|
||||
return sets;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fill(int fromIndex, int toIndex, long val) {
|
||||
assert val == (val%s);
|
||||
Arrays.fill(values, fromIndex, toIndex, %sval);
|
||||
}
|
||||
""" % (MASKS[bpv], CASTS[bpv], MASKS[bpv], CASTS[bpv]))
|
||||
|
||||
f.write("}\n")
|
||||
|
||||
f.close()
|
||||
|
|
@ -80,7 +80,7 @@ abstract class Packed64SingleBlock extends PackedInts.MutableImpl {
|
|||
public long ramBytesUsed() {
|
||||
return RamUsageEstimator.alignObjectSize(
|
||||
RamUsageEstimator.NUM_BYTES_OBJECT_HEADER
|
||||
+ 2 * RamUsageEstimator.NUM_BYTES_INT // valueCount,bitsPerValue
|
||||
+ 2 * Integer.BYTES // valueCount,bitsPerValue
|
||||
+ RamUsageEstimator.NUM_BYTES_OBJECT_REF) // blocks ref
|
||||
+ RamUsageEstimator.sizeOf(blocks);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,162 +0,0 @@
|
|||
#! /usr/bin/env python
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
HEADER = """// This file has been automatically generated, DO NOT EDIT
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.lucene.util.packed;
|
||||
|
||||
import org.apache.lucene.store.DataInput;
|
||||
import org.apache.lucene.util.RamUsageEstimator;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
|
||||
"""
|
||||
|
||||
TYPES = {8: "byte", 16: "short"}
|
||||
MASKS = {8: " & 0xFFL", 16: " & 0xFFFFL", 32: " & 0xFFFFFFFFL", 64: ""}
|
||||
CASTS = {8: "(byte) ", 16: "(short) ", 32: "(int) ", 64: ""}
|
||||
|
||||
if __name__ == '__main__':
|
||||
for bpv in TYPES.keys():
|
||||
type
|
||||
f = open("Packed%dThreeBlocks.java" % bpv, 'w')
|
||||
f.write(HEADER)
|
||||
f.write("""/**
|
||||
* Packs integers into 3 %ss (%d bits per value).
|
||||
* @lucene.internal
|
||||
*/\n""" % (TYPES[bpv], bpv * 3))
|
||||
f.write("final class Packed%dThreeBlocks extends PackedInts.MutableImpl {\n" % bpv)
|
||||
f.write(" final %s[] blocks;\n\n" % TYPES[bpv])
|
||||
|
||||
f.write(" public static final int MAX_SIZE = Integer.MAX_VALUE / 3;\n\n")
|
||||
|
||||
f.write(" Packed%dThreeBlocks(int valueCount) {\n" % bpv)
|
||||
f.write(" super(valueCount, %d);\n" % (bpv * 3))
|
||||
f.write(" if (valueCount > MAX_SIZE) {\n")
|
||||
f.write(" throw new ArrayIndexOutOfBoundsException(\"MAX_SIZE exceeded\");\n")
|
||||
f.write(" }\n")
|
||||
f.write(" blocks = new %s[valueCount * 3];\n" % TYPES[bpv])
|
||||
f.write(" }\n\n")
|
||||
|
||||
f.write(" Packed%dThreeBlocks(int packedIntsVersion, DataInput in, int valueCount) throws IOException {\n" % bpv)
|
||||
f.write(" this(valueCount);\n")
|
||||
if bpv == 8:
|
||||
f.write(" in.readBytes(blocks, 0, 3 * valueCount);\n")
|
||||
else:
|
||||
f.write(" for (int i = 0; i < 3 * valueCount; ++i) {\n")
|
||||
f.write(" blocks[i] = in.read%s();\n" % TYPES[bpv].title())
|
||||
f.write(" }\n")
|
||||
f.write(" }\n")
|
||||
|
||||
f.write("""
|
||||
@Override
|
||||
public long get(int index) {
|
||||
final int o = index * 3;
|
||||
return (blocks[o]%s) << %d | (blocks[o+1]%s) << %d | (blocks[o+2]%s);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int get(int index, long[] arr, int off, int len) {
|
||||
assert len > 0 : "len must be > 0 (got " + len + ")";
|
||||
assert index >= 0 && index < valueCount;
|
||||
assert off + len <= arr.length;
|
||||
|
||||
final int gets = Math.min(valueCount - index, len);
|
||||
for (int i = index * 3, end = (index + gets) * 3; i < end; i+=3) {
|
||||
arr[off++] = (blocks[i]%s) << %d | (blocks[i+1]%s) << %d | (blocks[i+2]%s);
|
||||
}
|
||||
return gets;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void set(int index, long value) {
|
||||
final int o = index * 3;
|
||||
blocks[o] = %s(value >>> %d);
|
||||
blocks[o+1] = %s(value >>> %d);
|
||||
blocks[o+2] = %svalue;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int set(int index, long[] arr, int off, int len) {
|
||||
assert len > 0 : "len must be > 0 (got " + len + ")";
|
||||
assert index >= 0 && index < valueCount;
|
||||
assert off + len <= arr.length;
|
||||
|
||||
final int sets = Math.min(valueCount - index, len);
|
||||
for (int i = off, o = index * 3, end = off + sets; i < end; ++i) {
|
||||
final long value = arr[i];
|
||||
blocks[o++] = %s(value >>> %d);
|
||||
blocks[o++] = %s(value >>> %d);
|
||||
blocks[o++] = %svalue;
|
||||
}
|
||||
return sets;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fill(int fromIndex, int toIndex, long val) {
|
||||
final %s block1 = %s(val >>> %d);
|
||||
final %s block2 = %s(val >>> %d);
|
||||
final %s block3 = %sval;
|
||||
for (int i = fromIndex * 3, end = toIndex * 3; i < end; i += 3) {
|
||||
blocks[i] = block1;
|
||||
blocks[i+1] = block2;
|
||||
blocks[i+2] = block3;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void clear() {
|
||||
Arrays.fill(blocks, %s0);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long ramBytesUsed() {
|
||||
return RamUsageEstimator.alignObjectSize(
|
||||
RamUsageEstimator.NUM_BYTES_OBJECT_HEADER
|
||||
+ 2 * RamUsageEstimator.NUM_BYTES_INT // valueCount,bitsPerValue
|
||||
+ RamUsageEstimator.NUM_BYTES_OBJECT_REF) // blocks ref
|
||||
+ RamUsageEstimator.sizeOf(blocks);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return getClass().getSimpleName() + "(bitsPerValue=" + bitsPerValue
|
||||
+ ",size=" + size() + ",blocks=" + blocks.length + ")";
|
||||
}
|
||||
}
|
||||
""" % (MASKS[bpv], 2 * bpv, MASKS[bpv], bpv, MASKS[bpv], MASKS[bpv], 2 * bpv, MASKS[bpv], bpv, MASKS[bpv], CASTS[bpv], 2 * bpv, CASTS[bpv], bpv, CASTS[bpv], CASTS[bpv],
|
||||
2 * bpv, CASTS[bpv], bpv, CASTS[bpv], TYPES[bpv], CASTS[bpv], 2 * bpv, TYPES[bpv],
|
||||
CASTS[bpv], bpv, TYPES[bpv], CASTS[bpv], CASTS[bpv]))
|
||||
|
||||
f.close()
|
||||
|
|
@ -35,7 +35,7 @@ com.fasterxml.jackson.core.version = 2.10.1
|
|||
/com.googlecode.mp4parser/isoparser = 1.1.22
|
||||
/com.healthmarketscience.jackcess/jackcess = 3.0.1
|
||||
/com.healthmarketscience.jackcess/jackcess-encrypt = 3.0.0
|
||||
/com.ibm.icu/icu4j = 62.1
|
||||
/com.ibm.icu/icu4j = 62.2
|
||||
/com.jayway.jsonpath/json-path = 2.4.0
|
||||
/com.lmax/disruptor = 3.4.2
|
||||
/com.pff/java-libpst = 0.8.1
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
7a4d00d5ec5febd252a6182e8b6e87a0a9821f81
|
||||
|
|
@ -0,0 +1 @@
|
|||
9ad0d915018dcbb394678a920d72f606cd1c7214
|
||||
|
|
@ -112,4 +112,4 @@ interface CharStream {
|
|||
void Done();
|
||||
|
||||
}
|
||||
/* JavaCC - OriginalChecksum=30b94cad7b10d0d81e3a59a1083939d0 (do not edit this line) */
|
||||
/* JavaCC - OriginalChecksum=6a5b704d1e6e4e036de8c95b24d47787 (do not edit this line) */
|
||||
|
|
|
|||
|
|
@ -173,7 +173,7 @@ public class ParseException extends Exception {
|
|||
default:
|
||||
if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
|
||||
String s = "0000" + Integer.toString(ch, 16);
|
||||
retval.append("\\u").append(s.substring(s.length() - 4, s.length()));
|
||||
retval.append("\\u" + s.substring(s.length() - 4, s.length()));
|
||||
} else {
|
||||
retval.append(ch);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ import org.apache.lucene.search.TermRangeQuery;
|
|||
* the same syntax as this class, but is more modular,
|
||||
* enabling substantial customization to how a query is created.
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
public class QueryParser extends QueryParserBase implements QueryParserConstants {
|
||||
/** The default operator for parsing queries.
|
||||
* Use {@link QueryParserBase#setDefaultOperator} to change it.
|
||||
|
|
@ -863,7 +864,7 @@ public class QueryParser extends QueryParserBase implements QueryParserConstants
|
|||
return (jj_ntk = jj_nt.kind);
|
||||
}
|
||||
|
||||
private java.util.List<int[]> jj_expentries = new java.util.ArrayList<int[]>();
|
||||
private java.util.List jj_expentries = new java.util.ArrayList();
|
||||
private int[] jj_expentry;
|
||||
private int jj_kind = -1;
|
||||
private int[] jj_lasttokens = new int[100];
|
||||
|
|
@ -878,7 +879,7 @@ public class QueryParser extends QueryParserBase implements QueryParserConstants
|
|||
for (int i = 0; i < jj_endpos; i++) {
|
||||
jj_expentry[i] = jj_lasttokens[i];
|
||||
}
|
||||
jj_entries_loop: for (java.util.Iterator<?> it = jj_expentries.iterator(); it.hasNext();) {
|
||||
jj_entries_loop: for (java.util.Iterator it = jj_expentries.iterator(); it.hasNext();) {
|
||||
int[] oldentry = (int[])(it.next());
|
||||
if (oldentry.length == jj_expentry.length) {
|
||||
for (int i = 0; i < jj_expentry.length; i++) {
|
||||
|
|
@ -926,7 +927,7 @@ public class QueryParser extends QueryParserBase implements QueryParserConstants
|
|||
jj_add_error_token(0, 0);
|
||||
int[][] exptokseq = new int[jj_expentries.size()][];
|
||||
for (int i = 0; i < jj_expentries.size(); i++) {
|
||||
exptokseq[i] = jj_expentries.get(i);
|
||||
exptokseq[i] = (int[])jj_expentries.get(i);
|
||||
}
|
||||
return new ParseException(token, exptokseq, tokenImage);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ public class TokenMgrError extends Error
|
|||
default:
|
||||
if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
|
||||
String s = "0000" + Integer.toString(ch, 16);
|
||||
retval.append("\\u").append(s.substring(s.length() - 4, s.length()));
|
||||
retval.append("\\u" + s.substring(s.length() - 4, s.length()));
|
||||
} else {
|
||||
retval.append(ch);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -112,4 +112,4 @@ interface CharStream {
|
|||
void Done();
|
||||
|
||||
}
|
||||
/* JavaCC - OriginalChecksum=c95f1720d9b38046dc5d294b741c44cb (do not edit this line) */
|
||||
/* JavaCC - OriginalChecksum=ad6e2aadb6864b25f5ca6701803e4f89 (do not edit this line) */
|
||||
|
|
|
|||
|
|
@ -176,7 +176,7 @@ public class ParseException extends QueryNodeParseException {
|
|||
default:
|
||||
if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
|
||||
String s = "0000" + Integer.toString(ch, 16);
|
||||
retval.append("\\u").append(s.substring(s.length() - 4, s.length()));
|
||||
retval.append("\\u" + s.substring(s.length() - 4, s.length()));
|
||||
} else {
|
||||
retval.append(ch);
|
||||
}
|
||||
|
|
@ -187,4 +187,4 @@ public class ParseException extends QueryNodeParseException {
|
|||
}
|
||||
|
||||
}
|
||||
/* JavaCC - OriginalChecksum=81401c29cf6f9909761c636b4778ccc0 (do not edit this line) */
|
||||
/* JavaCC - OriginalChecksum=4263a02db9988d7a863aa97ad2f6dc67 (do not edit this line) */
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ import org.apache.lucene.queryparser.flexible.standard.nodes.TermRangeQueryNode;
|
|||
/**
|
||||
* Parser for the standard Lucene syntax
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
public class StandardSyntaxParser implements SyntaxParser, StandardSyntaxParserConstants {
|
||||
|
||||
|
||||
|
|
@ -961,7 +962,7 @@ public class StandardSyntaxParser implements SyntaxParser, StandardSyntaxParserC
|
|||
return (jj_ntk = jj_nt.kind);
|
||||
}
|
||||
|
||||
private java.util.List<int[]> jj_expentries = new java.util.ArrayList<int[]>();
|
||||
private java.util.List jj_expentries = new java.util.ArrayList();
|
||||
private int[] jj_expentry;
|
||||
private int jj_kind = -1;
|
||||
private int[] jj_lasttokens = new int[100];
|
||||
|
|
@ -976,7 +977,7 @@ public class StandardSyntaxParser implements SyntaxParser, StandardSyntaxParserC
|
|||
for (int i = 0; i < jj_endpos; i++) {
|
||||
jj_expentry[i] = jj_lasttokens[i];
|
||||
}
|
||||
jj_entries_loop: for (java.util.Iterator<?> it = jj_expentries.iterator(); it.hasNext();) {
|
||||
jj_entries_loop: for (java.util.Iterator it = jj_expentries.iterator(); it.hasNext();) {
|
||||
int[] oldentry = (int[])(it.next());
|
||||
if (oldentry.length == jj_expentry.length) {
|
||||
for (int i = 0; i < jj_expentry.length; i++) {
|
||||
|
|
@ -1024,7 +1025,7 @@ public class StandardSyntaxParser implements SyntaxParser, StandardSyntaxParserC
|
|||
jj_add_error_token(0, 0);
|
||||
int[][] exptokseq = new int[jj_expentries.size()][];
|
||||
for (int i = 0; i < jj_expentries.size(); i++) {
|
||||
exptokseq[i] = jj_expentries.get(i);
|
||||
exptokseq[i] = (int[])jj_expentries.get(i);
|
||||
}
|
||||
return new ParseException(token, exptokseq, tokenImage);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -128,4 +128,4 @@ public class Token implements java.io.Serializable {
|
|||
}
|
||||
|
||||
}
|
||||
/* JavaCC - OriginalChecksum=30bbd23e0dec26f141130dc62a4f6e9d (do not edit this line) */
|
||||
/* JavaCC - OriginalChecksum=ea8b1e55950603be28e2f63dcd544ab4 (do not edit this line) */
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ public class TokenMgrError extends Error
|
|||
default:
|
||||
if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
|
||||
String s = "0000" + Integer.toString(ch, 16);
|
||||
retval.append("\\u").append(s.substring(s.length() - 4, s.length()));
|
||||
retval.append("\\u" + s.substring(s.length() - 4, s.length()));
|
||||
} else {
|
||||
retval.append(ch);
|
||||
}
|
||||
|
|
@ -144,4 +144,4 @@ public class TokenMgrError extends Error
|
|||
this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
|
||||
}
|
||||
}
|
||||
/* JavaCC - OriginalChecksum=3ca7fbf7de9f2424b131a5499b0a78d0 (do not edit this line) */
|
||||
/* JavaCC - OriginalChecksum=be88283d82a985d82a34dda46bcf42d5 (do not edit this line) */
|
||||
|
|
|
|||
|
|
@ -112,4 +112,4 @@ interface CharStream {
|
|||
void Done();
|
||||
|
||||
}
|
||||
/* JavaCC - OriginalChecksum=5ca20c9145f29a0f8909470a7f949fe4 (do not edit this line) */
|
||||
/* JavaCC - OriginalChecksum=a2f9487333c63c295c1dfad0d1fc9019 (do not edit this line) */
|
||||
|
|
|
|||
|
|
@ -173,7 +173,7 @@ public class ParseException extends Exception {
|
|||
default:
|
||||
if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
|
||||
String s = "0000" + Integer.toString(ch, 16);
|
||||
retval.append("\\u").append(s.substring(s.length() - 4, s.length()));
|
||||
retval.append("\\u" + s.substring(s.length() - 4, s.length()));
|
||||
} else {
|
||||
retval.append(ch);
|
||||
}
|
||||
|
|
@ -184,4 +184,4 @@ public class ParseException extends Exception {
|
|||
}
|
||||
|
||||
}
|
||||
/* JavaCC - OriginalChecksum=be6f55e3bf157e8c96b4c06cca5ec81b (do not edit this line) */
|
||||
/* JavaCC - OriginalChecksum=bd8163f41bf2fd1bb00f025fce3dcaaf (do not edit this line) */
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ import org.apache.lucene.queryparser.surround.query.SrndTruncQuery;
|
|||
* must appear within three positions of each other, or in other words, up
|
||||
* to two terms may appear between a and b. </p>
|
||||
*/
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public class QueryParser implements QueryParserConstants {
|
||||
static final int MINIMUM_PREFIX_LENGTH = 3;
|
||||
static final int MINIMUM_CHARS_IN_TRUNC = 3;
|
||||
|
|
@ -639,7 +639,7 @@ public class QueryParser implements QueryParserConstants {
|
|||
return (jj_ntk = jj_nt.kind);
|
||||
}
|
||||
|
||||
private java.util.List<int[]> jj_expentries = new java.util.ArrayList<int[]>();
|
||||
private java.util.List jj_expentries = new java.util.ArrayList();
|
||||
private int[] jj_expentry;
|
||||
private int jj_kind = -1;
|
||||
private int[] jj_lasttokens = new int[100];
|
||||
|
|
@ -654,7 +654,7 @@ public class QueryParser implements QueryParserConstants {
|
|||
for (int i = 0; i < jj_endpos; i++) {
|
||||
jj_expentry[i] = jj_lasttokens[i];
|
||||
}
|
||||
jj_entries_loop: for (java.util.Iterator<?> it = jj_expentries.iterator(); it.hasNext();) {
|
||||
jj_entries_loop: for (java.util.Iterator it = jj_expentries.iterator(); it.hasNext();) {
|
||||
int[] oldentry = (int[])(it.next());
|
||||
if (oldentry.length == jj_expentry.length) {
|
||||
for (int i = 0; i < jj_expentry.length; i++) {
|
||||
|
|
@ -699,7 +699,7 @@ public class QueryParser implements QueryParserConstants {
|
|||
jj_add_error_token(0, 0);
|
||||
int[][] exptokseq = new int[jj_expentries.size()][];
|
||||
for (int i = 0; i < jj_expentries.size(); i++) {
|
||||
exptokseq[i] = jj_expentries.get(i);
|
||||
exptokseq[i] = (int[])jj_expentries.get(i);
|
||||
}
|
||||
return new ParseException(token, exptokseq, tokenImage);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -128,4 +128,4 @@ public class Token implements java.io.Serializable {
|
|||
}
|
||||
|
||||
}
|
||||
/* JavaCC - OriginalChecksum=db38f23b3674db52ff034369707a0ac3 (do not edit this line) */
|
||||
/* JavaCC - OriginalChecksum=f2df701e24da1cf2d025118ce6efdd2f (do not edit this line) */
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ public class TokenMgrError extends Error
|
|||
default:
|
||||
if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
|
||||
String s = "0000" + Integer.toString(ch, 16);
|
||||
retval.append("\\u").append(s.substring(s.length() - 4, s.length()));
|
||||
retval.append("\\u" + s.substring(s.length() - 4, s.length()));
|
||||
} else {
|
||||
retval.append(ch);
|
||||
}
|
||||
|
|
@ -144,4 +144,4 @@ public class TokenMgrError extends Error
|
|||
this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
|
||||
}
|
||||
}
|
||||
/* JavaCC - OriginalChecksum=dcdd5ccde13b91bcd8f76a86ca618852 (do not edit this line) */
|
||||
/* JavaCC - OriginalChecksum=8c69a370d9a9893140562c8bb911678c (do not edit this line) */
|
||||
|
|
|
|||
|
|
@ -95,6 +95,8 @@ Other Changes
|
|||
|
||||
* LUCENE-9092: Upgrade Carrot2 to 3.16.2 (Dawid Weiss).
|
||||
|
||||
* LUCENE-9080: Upgrade ICU4j to 62.2 and make regenerate work (Erick Erickson)
|
||||
|
||||
================== 8.5.0 ==================
|
||||
|
||||
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
|
||||
|
|
@ -212,6 +214,8 @@ Bug Fixes
|
|||
|
||||
* SOLR-14163: SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION needs to work with Jetty server/client SSL contexts (Kevin Risden)
|
||||
|
||||
* SOLR-6613: TextField.analyzeMultiTerm does not throw an exception when Analyzer returns no terms. (Bruno Roustant)
|
||||
|
||||
Other Changes
|
||||
---------------------
|
||||
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -45,6 +45,7 @@ import org.apache.lucene.search.Query;
|
|||
import org.apache.lucene.search.QueryVisitor;
|
||||
import org.apache.lucene.search.RegexpQuery;
|
||||
import org.apache.lucene.search.WildcardQuery;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.QueryBuilder;
|
||||
import org.apache.lucene.util.automaton.Automata;
|
||||
import org.apache.lucene.util.automaton.Automaton;
|
||||
|
|
@ -997,8 +998,8 @@ public abstract class SolrQueryParserBase extends QueryBuilder {
|
|||
|
||||
SchemaField sf = schema.getFieldOrNull((field));
|
||||
if (sf == null || ! (fieldType instanceof TextField)) return part;
|
||||
String out = TextField.analyzeMultiTerm(field, part, ((TextField)fieldType).getMultiTermAnalyzer()).utf8ToString();
|
||||
return out;
|
||||
BytesRef out = TextField.analyzeMultiTerm(field, part, ((TextField)fieldType).getMultiTermAnalyzer());
|
||||
return out == null ? part : out.utf8ToString();
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -165,6 +165,16 @@ public class TextField extends FieldType {
|
|||
return new SolrRangeQuery(field.getName(), lower, upper, minInclusive, maxInclusive);
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyzes a text part using the provided {@link Analyzer} for a multi-term query.
|
||||
* <p>
|
||||
* Expects a single token to be used as multi-term term. This single token might also be filtered out
|
||||
* so zero token is supported and null is returned in this case.
|
||||
*
|
||||
* @return The multi-term term bytes; or null if there is no multi-term terms.
|
||||
* @throws SolrException If the {@link Analyzer} tokenizes more than one token;
|
||||
* or if an underlying {@link IOException} occurs.
|
||||
*/
|
||||
public static BytesRef analyzeMultiTerm(String field, String part, Analyzer analyzerIn) {
|
||||
if (part == null || analyzerIn == null) return null;
|
||||
|
||||
|
|
@ -173,8 +183,10 @@ public class TextField extends FieldType {
|
|||
|
||||
TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class);
|
||||
|
||||
if (!source.incrementToken())
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"analyzer returned no terms for multiTerm term: " + part);
|
||||
if (!source.incrementToken()) {
|
||||
// Accept no tokens because it may have been filtered out by a StopFilter for example.
|
||||
return null;
|
||||
}
|
||||
BytesRef bytes = BytesRef.deepCopyOf(termAtt.getBytesRef());
|
||||
if (source.incrementToken())
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"analyzer returned too many terms for multiTerm term: " + part);
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import org.apache.lucene.search.BooleanQuery;
|
|||
import org.apache.lucene.search.BoostQuery;
|
||||
import org.apache.lucene.search.FuzzyQuery;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.SimpleParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
|
|
@ -186,25 +187,29 @@ public class SimpleQParserPlugin extends QParserPlugin {
|
|||
for (Map.Entry<String, Float> entry : weights.entrySet()) {
|
||||
String field = entry.getKey();
|
||||
FieldType type = schema.getFieldType(field);
|
||||
Query prefix;
|
||||
Query prefix = null;
|
||||
|
||||
if (type instanceof TextField) {
|
||||
// If the field type is a TextField then use the multi term analyzer.
|
||||
Analyzer analyzer = ((TextField)type).getMultiTermAnalyzer();
|
||||
String term = TextField.analyzeMultiTerm(field, text, analyzer).utf8ToString();
|
||||
SchemaField sf = schema.getField(field);
|
||||
prefix = sf.getType().getPrefixQuery(qParser, sf, term);
|
||||
BytesRef termBytes = TextField.analyzeMultiTerm(field, text, analyzer);
|
||||
if (termBytes != null) {
|
||||
String term = termBytes.utf8ToString();
|
||||
SchemaField sf = schema.getField(field);
|
||||
prefix = sf.getType().getPrefixQuery(qParser, sf, term);
|
||||
}
|
||||
} else {
|
||||
// If the type is *not* a TextField don't do any analysis.
|
||||
SchemaField sf = schema.getField(field);
|
||||
prefix = type.getPrefixQuery(qParser, sf, text);
|
||||
}
|
||||
|
||||
float boost = entry.getValue();
|
||||
if (boost != 1f) {
|
||||
prefix = new BoostQuery(prefix, boost);
|
||||
if (prefix != null) {
|
||||
float boost = entry.getValue();
|
||||
if (boost != 1f) {
|
||||
prefix = new BoostQuery(prefix, boost);
|
||||
}
|
||||
bq.add(prefix, BooleanClause.Occur.SHOULD);
|
||||
}
|
||||
bq.add(prefix, BooleanClause.Occur.SHOULD);
|
||||
}
|
||||
|
||||
return simplify(bq.build());
|
||||
|
|
@ -217,23 +222,27 @@ public class SimpleQParserPlugin extends QParserPlugin {
|
|||
for (Map.Entry<String, Float> entry : weights.entrySet()) {
|
||||
String field = entry.getKey();
|
||||
FieldType type = schema.getFieldType(field);
|
||||
Query fuzzy;
|
||||
Query fuzzy = null;
|
||||
|
||||
if (type instanceof TextField) {
|
||||
// If the field type is a TextField then use the multi term analyzer.
|
||||
Analyzer analyzer = ((TextField)type).getMultiTermAnalyzer();
|
||||
String term = TextField.analyzeMultiTerm(field, text, analyzer).utf8ToString();
|
||||
fuzzy = new FuzzyQuery(new Term(entry.getKey(), term), fuzziness);
|
||||
BytesRef termBytes = TextField.analyzeMultiTerm(field, text, analyzer);
|
||||
if (termBytes != null) {
|
||||
String term = termBytes.utf8ToString();
|
||||
fuzzy = new FuzzyQuery(new Term(entry.getKey(), term), fuzziness);
|
||||
}
|
||||
} else {
|
||||
// If the type is *not* a TextField don't do any analysis.
|
||||
fuzzy = new FuzzyQuery(new Term(entry.getKey(), text), fuzziness);
|
||||
}
|
||||
|
||||
float boost = entry.getValue();
|
||||
if (boost != 1f) {
|
||||
fuzzy = new BoostQuery(fuzzy, boost);
|
||||
if (fuzzy != null) {
|
||||
float boost = entry.getValue();
|
||||
if (boost != 1f) {
|
||||
fuzzy = new BoostQuery(fuzzy, boost);
|
||||
}
|
||||
bq.add(fuzzy, BooleanClause.Occur.SHOULD);
|
||||
}
|
||||
bq.add(fuzzy, BooleanClause.Occur.SHOULD);
|
||||
}
|
||||
|
||||
return simplify(bq.build());
|
||||
|
|
|
|||
|
|
@ -140,7 +140,7 @@ public class SolrLogPostTool {
|
|||
private String pushedBack = null;
|
||||
private boolean finished = false;
|
||||
private String cause;
|
||||
private Pattern p = Pattern.compile("^(\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d \\d\\d:\\d\\d\\:\\d\\d.\\d\\d\\d)");
|
||||
private Pattern p = Pattern.compile("^(\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d[\\s|T]\\d\\d:\\d\\d\\:\\d\\d.\\d\\d\\d)");
|
||||
|
||||
public LogRecordReader(BufferedReader bufferedReader) throws IOException {
|
||||
this.bufferedReader = bufferedReader;
|
||||
|
|
@ -314,7 +314,7 @@ public class SolrLogPostTool {
|
|||
}
|
||||
|
||||
private String parseCollection(String line) {
|
||||
char[] ca = {' ', ']'};
|
||||
char[] ca = {' ', ']', ','};
|
||||
String parts[] = line.split("c:");
|
||||
if(parts.length >= 2) {
|
||||
return readUntil(parts[1], ca);
|
||||
|
|
@ -355,7 +355,7 @@ public class SolrLogPostTool {
|
|||
}
|
||||
|
||||
private String parseCore(String line) {
|
||||
char[] ca = {' ', ']'};
|
||||
char[] ca = {' ', ']', '}', ','};
|
||||
String parts[] = line.split("x:");
|
||||
if(parts.length >= 2) {
|
||||
return readUntil(parts[1], ca);
|
||||
|
|
@ -365,7 +365,7 @@ public class SolrLogPostTool {
|
|||
}
|
||||
|
||||
private String parseShard(String line) {
|
||||
char[] ca = {' ', ']'};
|
||||
char[] ca = {' ', ']', '}', ','};
|
||||
String parts[] = line.split("s:");
|
||||
if(parts.length >= 2) {
|
||||
return readUntil(parts[1], ca);
|
||||
|
|
@ -375,7 +375,7 @@ public class SolrLogPostTool {
|
|||
}
|
||||
|
||||
private String parseReplica(String line) {
|
||||
char[] ca = {' ', ']'};
|
||||
char[] ca = {' ', ']', '}',','};
|
||||
String parts[] = line.split("r:");
|
||||
if(parts.length >= 2) {
|
||||
return readUntil(parts[1], ca);
|
||||
|
|
@ -406,9 +406,9 @@ public class SolrLogPostTool {
|
|||
}
|
||||
|
||||
private String parseNode(String line) {
|
||||
char[] ca = {' ', ']'};
|
||||
String parts[] = line.split("n:");
|
||||
if(parts.length == 2) {
|
||||
char[] ca = {' ', ']', '}', ','};
|
||||
String parts[] = line.split("node_name=n:");
|
||||
if(parts.length >= 2) {
|
||||
return readUntil(parts[1], ca);
|
||||
} else {
|
||||
return null;
|
||||
|
|
@ -436,10 +436,11 @@ public class SolrLogPostTool {
|
|||
}
|
||||
|
||||
private String parseParams(String line) {
|
||||
char[] ca = {'}'};
|
||||
char[] ca = {' '};
|
||||
String parts[] = line.split(" params=");
|
||||
if(parts.length == 2) {
|
||||
return readUntil(parts[1].substring(1), ca);
|
||||
String p = readUntil(parts[1].substring(1), ca);
|
||||
return p.substring(0, p.length()-1);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,11 +48,10 @@ import org.junit.Before;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
@AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13486")
|
||||
@AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13486;https://issues.apache.org/jira/browse/SOLR-14183")
|
||||
public class TestTlogReplayVsRecovery extends SolrCloudTestCase {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
||||
private static final String COLLECTION = "collecion_with_slow_tlog_recovery";
|
||||
|
||||
private JettySolrRunner NODE0;
|
||||
|
|
@ -60,12 +59,16 @@ public class TestTlogReplayVsRecovery extends SolrCloudTestCase {
|
|||
private Map<JettySolrRunner, SocketProxy> proxies;
|
||||
private Map<URI, JettySolrRunner> jettys;
|
||||
|
||||
// we want to ensure there is tlog replay on the leader after we restart it,
|
||||
// so in addition to not committing the docs we add during network partition
|
||||
// we also want to ensure that our leader doesn't do a "Commit on close"
|
||||
//
|
||||
// TODO: once SOLR-13486 is fixed, we should randomize this...
|
||||
private static final boolean TEST_VALUE_FOR_COMMIT_ON_CLOSE = false;
|
||||
|
||||
@Before
|
||||
public void setupCluster() throws Exception {
|
||||
// we want to ensure there is tlog replay on the leader after we restart it,
|
||||
// so in addition to not committing the docs we add during network partition
|
||||
// we also want to ensure that our leader doesn't do a "Commit on close"
|
||||
DirectUpdateHandler2.commitOnClose = false;
|
||||
DirectUpdateHandler2.commitOnClose = TEST_VALUE_FOR_COMMIT_ON_CLOSE;
|
||||
|
||||
System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
|
||||
System.setProperty("solr.ulog.numRecordsToKeep", "1000");
|
||||
|
|
@ -113,6 +116,14 @@ public class TestTlogReplayVsRecovery extends SolrCloudTestCase {
|
|||
}
|
||||
|
||||
public void testManyDocsInTlogReplayWhileReplicaIsTryingToRecover() throws Exception {
|
||||
// TODO: One the basic problem in SOLR-13486 is fixed, this test can be made more robust by:
|
||||
// 1) randomizing the number of committedDocs (pre net split) & uncommittedDocs (post net split)
|
||||
// to trigger diff recovery strategies & shutdown behavior
|
||||
// 2) replace "committedDocs + uncommittedDocs" with 4 variables:
|
||||
// a: docs committed before network split (add + commit)
|
||||
// b: docs not committed before network split (add w/o commit)
|
||||
// c: docs committed after network split (add + commit)
|
||||
// d: docs not committed after network split (add w/o commit)
|
||||
final int committedDocs = 3;
|
||||
final int uncommittedDocs = 50;
|
||||
|
||||
|
|
@ -140,8 +151,9 @@ public class TestTlogReplayVsRecovery extends SolrCloudTestCase {
|
|||
final Replica leader = getCollectionState(COLLECTION).getSlice("shard1").getLeader();
|
||||
assertEquals("Sanity check failed", NODE0.getNodeName(), leader.getNodeName());
|
||||
|
||||
log.info("Add and commit a {} docs...", committedDocs);
|
||||
log.info("Add and commit {} docs...", committedDocs);
|
||||
addDocs(true, committedDocs, 1);
|
||||
assertDocsExistInBothReplicas(1, committedDocs);
|
||||
|
||||
log.info("Partition nodes...");
|
||||
proxies.get(NODE0).close();
|
||||
|
|
@ -151,7 +163,8 @@ public class TestTlogReplayVsRecovery extends SolrCloudTestCase {
|
|||
addDocs(false, uncommittedDocs, committedDocs + 1);
|
||||
|
||||
log.info("Stopping leader node...");
|
||||
assertEquals("Something broke our expected commitOnClose", false, DirectUpdateHandler2.commitOnClose);
|
||||
assertEquals("Something broke our expected commitOnClose",
|
||||
TEST_VALUE_FOR_COMMIT_ON_CLOSE, DirectUpdateHandler2.commitOnClose);
|
||||
NODE0.stop();
|
||||
cluster.waitForJettyToStop(NODE0);
|
||||
|
||||
|
|
@ -188,7 +201,7 @@ public class TestTlogReplayVsRecovery extends SolrCloudTestCase {
|
|||
cluster.waitForActiveCollection(COLLECTION, 1, 2);
|
||||
|
||||
log.info("Check docs on both replicas...");
|
||||
assertDocsExistInBothReplicas(1, uncommittedDocs + uncommittedDocs);
|
||||
assertDocsExistInBothReplicas(1, committedDocs + uncommittedDocs);
|
||||
|
||||
log.info("Test ok, delete collection...");
|
||||
CollectionAdminRequest.deleteCollection(COLLECTION).process(cluster.getSolrClient());
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.schema;
|
||||
|
||||
import org.apache.lucene.analysis.core.StopAnalyzer;
|
||||
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
|
||||
import org.apache.lucene.analysis.en.EnglishAnalyzer;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
* Tests directly {@link org.apache.solr.schema.TextField} methods.
|
||||
*/
|
||||
public class TestTextField extends SolrTestCaseJ4 {
|
||||
|
||||
@Test
|
||||
public void testAnalyzeMultiTerm() {
|
||||
// No terms provided by the StopFilter (stop word) for the multi-term part.
|
||||
// This is supported. Check TextField.analyzeMultiTerm returns null (and does not throw an exception).
|
||||
BytesRef termBytes = TextField.analyzeMultiTerm("field", "the", new StopAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET));
|
||||
assertNull(termBytes);
|
||||
|
||||
// One term provided by the WhitespaceTokenizer for the multi-term part.
|
||||
// This is the regular case. Check TextField.analyzeMultiTerm returns it (and does not throw an exception).
|
||||
termBytes = TextField.analyzeMultiTerm("field", "Sol", new WhitespaceAnalyzer());
|
||||
assertEquals("Sol", termBytes.utf8ToString());
|
||||
|
||||
// Two terms provided by the WhitespaceTokenizer for the multi-term part.
|
||||
// This is not allowed. Expect an exception.
|
||||
SolrException exception = expectThrows(SolrException.class, () -> TextField.analyzeMultiTerm("field", "term1 term2", new WhitespaceAnalyzer()));
|
||||
assertEquals("Unexpected error code", SolrException.ErrorCode.BAD_REQUEST.code, exception.code());
|
||||
}
|
||||
}
|
||||
|
|
@ -201,7 +201,7 @@ public class SolrLogPostToolTest extends SolrTestCaseJ4 {
|
|||
|
||||
@Test
|
||||
public void testCommit() throws Exception{
|
||||
String record = "2019-12-16 14:20:19.708 INFO (qtp812143047-22671) [c:production_201912 s:shard128 r:core_node7 x:production_201912_shard128_replica] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1653086376121335808,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}\n";
|
||||
String record = "2019-12-16T14:20:19.708 INFO (qtp812143047-22671) [c:production_201912 s:shard128 r:core_node7 x:production_201912_shard128_replica] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1653086376121335808,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}\n";
|
||||
List<SolrInputDocument> docs = readDocs(record);
|
||||
assertEquals(docs.size(), 1);
|
||||
SolrInputDocument doc = docs.get(0);
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
7a4d00d5ec5febd252a6182e8b6e87a0a9821f81
|
||||
|
|
@ -0,0 +1 @@
|
|||
9ad0d915018dcbb394678a920d72f606cd1c7214
|
||||
Loading…
Reference in New Issue