From 760f2dbdcb29b993aab8f981d84ccbf2e20e9fa5 Mon Sep 17 00:00:00 2001 From: Michael Sokolov Date: Thu, 4 Jul 2019 09:45:51 -0400 Subject: [PATCH 01/38] LUCENE-8920: encapsulate FST.Arc data --- .../charfilter/MappingCharFilter.java | 8 +- .../analysis/charfilter/NormalizeCharMap.java | 6 +- .../lucene/analysis/hunspell/Dictionary.java | 12 +- .../lucene/analysis/hunspell/Stemmer.java | 12 +- .../miscellaneous/StemmerOverrideFilter.java | 4 +- .../analysis/synonym/SynonymFilter.java | 8 +- .../analysis/synonym/SynonymGraphFilter.java | 8 +- .../lucene/analysis/ja/JapaneseTokenizer.java | 8 +- .../analysis/ja/dict/UserDictionary.java | 4 +- .../lucene/analysis/ko/KoreanTokenizer.java | 8 +- .../analysis/ko/dict/UserDictionary.java | 4 +- .../blocktreeords/OrdsIntersectTermsEnum.java | 6 +- .../blocktreeords/OrdsSegmentTermsEnum.java | 74 ++-- .../codecs/memory/FSTOrdTermsReader.java | 28 +- .../lucene/codecs/memory/FSTTermsReader.java | 30 +- .../codecs/blocktree/IntersectTermsEnum.java | 6 +- .../codecs/blocktree/SegmentTermsEnum.java | 48 +-- .../java/org/apache/lucene/util/fst/FST.java | 376 ++++++++++-------- .../org/apache/lucene/util/fst/FSTEnum.java | 144 ++++--- .../org/apache/lucene/util/fst/NodeHash.java | 18 +- .../java/org/apache/lucene/util/fst/Util.java | 168 ++++---- .../org/apache/lucene/util/fst/TestFSTs.java | 20 +- .../apache/lucene/util/fst/TestFstDirect.java | 4 +- .../idversion/IDVersionSegmentTermsEnum.java | 48 +-- .../suggest/analyzing/AnalyzingSuggester.java | 2 +- .../search/suggest/analyzing/FSTUtil.java | 18 +- .../suggest/analyzing/FreeTextSuggester.java | 4 +- .../search/suggest/document/NRTSuggester.java | 2 +- .../search/suggest/fst/FSTCompletion.java | 10 +- .../suggest/fst/WFSTCompletionLookup.java | 6 +- .../org/apache/lucene/util/fst/FSTTester.java | 8 +- 31 files changed, 579 insertions(+), 523 deletions(-) diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java index 764e1c659f1..5fffda94030 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java @@ -106,17 +106,17 @@ public class MappingCharFilter extends BaseCharFilter { // Fast pass for single character match: assert arc.isFinal(); lastMatchLen = 1; - lastMatch = arc.output; + lastMatch = arc.output(); } else { int lookahead = 0; - CharsRef output = arc.output; + CharsRef output = arc.output(); while (true) { lookahead++; if (arc.isFinal()) { // Match! (to node is final) lastMatchLen = lookahead; - lastMatch = outputs.add(output, arc.nextFinalOutput); + lastMatch = outputs.add(output, arc.nextFinalOutput()); // Greedy: keep searching to see if there's a // longer match... } @@ -133,7 +133,7 @@ public class MappingCharFilter extends BaseCharFilter { // Dead end break; } - output = outputs.add(output, arc.output); + output = outputs.add(output, arc.output()); } } } diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/NormalizeCharMap.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/NormalizeCharMap.java index ffc832f267b..b3efcf73f86 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/NormalizeCharMap.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/NormalizeCharMap.java @@ -50,10 +50,10 @@ public class NormalizeCharMap { final FST.BytesReader fstReader = map.getBytesReader(); map.getFirstArc(scratchArc); if (FST.targetHasArcs(scratchArc)) { - map.readFirstRealTargetArc(scratchArc.target, scratchArc, fstReader); + map.readFirstRealTargetArc(scratchArc.target(), scratchArc, fstReader); while(true) { - assert scratchArc.label != FST.END_LABEL; - cachedRootArcs.put(Character.valueOf((char) scratchArc.label), new FST.Arc().copyFrom(scratchArc)); + assert scratchArc.label() != FST.END_LABEL; + cachedRootArcs.put(Character.valueOf((char) scratchArc.label()), new FST.Arc().copyFrom(scratchArc)); if (scratchArc.isLast()) { break; } diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java index 6af2f12b0e3..443f010047f 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java @@ -280,14 +280,14 @@ public class Dictionary { cp = Character.codePointAt(word, i, l); if (fst.findTargetArc(cp, arc, arc, bytesReader) == null) { return null; - } else if (arc.output != NO_OUTPUT) { - output = fst.outputs.add(output, arc.output); + } else if (arc.output() != NO_OUTPUT) { + output = fst.outputs.add(output, arc.output()); } } if (fst.findTargetArc(FST.END_LABEL, arc, arc, bytesReader) == null) { return null; - } else if (arc.output != NO_OUTPUT) { - return fst.outputs.add(output, arc.output); + } else if (arc.output() != NO_OUTPUT) { + return fst.outputs.add(output, arc.output()); } else { return output; } @@ -1228,10 +1228,10 @@ public class Dictionary { if (fst.findTargetArc(ch, arc, arc, bytesReader) == null) { break; } else { - output = fst.outputs.add(output, arc.output); + output = fst.outputs.add(output, arc.output()); } if (arc.isFinal()) { - longestOutput = fst.outputs.add(output, arc.nextFinalOutput); + longestOutput = fst.outputs.add(output, arc.nextFinalOutput()); longestMatch = j; } } diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java index a7b4e8b2092..888d89eb19c 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java @@ -311,15 +311,15 @@ final class Stemmer { int ch = word[i-1]; if (fst.findTargetArc(ch, arc, arc, bytesReader) == null) { break; - } else if (arc.output != NO_OUTPUT) { - output = fst.outputs.add(output, arc.output); + } else if (arc.output() != NO_OUTPUT) { + output = fst.outputs.add(output, arc.output()); } } IntsRef prefixes = null; if (!arc.isFinal()) { continue; } else { - prefixes = fst.outputs.add(output, arc.nextFinalOutput); + prefixes = fst.outputs.add(output, arc.nextFinalOutput()); } for (int j = 0; j < prefixes.length; j++) { @@ -395,15 +395,15 @@ final class Stemmer { int ch = word[i]; if (fst.findTargetArc(ch, arc, arc, bytesReader) == null) { break; - } else if (arc.output != NO_OUTPUT) { - output = fst.outputs.add(output, arc.output); + } else if (arc.output() != NO_OUTPUT) { + output = fst.outputs.add(output, arc.output()); } } IntsRef suffixes = null; if (!arc.isFinal()) { continue; } else { - suffixes = fst.outputs.add(output, arc.nextFinalOutput); + suffixes = fst.outputs.add(output, arc.nextFinalOutput()); } for (int j = 0; j < suffixes.length; j++) { diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java index 32423e96e93..078865fc344 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java @@ -132,11 +132,11 @@ public final class StemmerOverrideFilter extends TokenFilter { if (fst.findTargetArc(ignoreCase ? Character.toLowerCase(codePoint) : codePoint, scratchArc, scratchArc, fstReader) == null) { return null; } - pendingOutput = fst.outputs.add(pendingOutput, scratchArc.output); + pendingOutput = fst.outputs.add(pendingOutput, scratchArc.output()); bufUpto += Character.charCount(codePoint); } if (scratchArc.isFinal()) { - matchOutput = fst.outputs.add(pendingOutput, scratchArc.nextFinalOutput); + matchOutput = fst.outputs.add(pendingOutput, scratchArc.nextFinalOutput()); } return matchOutput; } diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java index a51edb5dba0..6894353fa12 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java @@ -330,7 +330,7 @@ public final class SynonymFilter extends TokenFilter { BytesRef pendingOutput = fst.outputs.getNoOutput(); fst.getFirstArc(scratchArc); - assert scratchArc.output == fst.outputs.getNoOutput(); + assert scratchArc.output() == fst.outputs.getNoOutput(); int tokenCount = 0; @@ -399,7 +399,7 @@ public final class SynonymFilter extends TokenFilter { } // Accum the output - pendingOutput = fst.outputs.add(pendingOutput, scratchArc.output); + pendingOutput = fst.outputs.add(pendingOutput, scratchArc.output()); //System.out.println(" char=" + buffer[bufUpto] + " output=" + pendingOutput + " arc.output=" + scratchArc.output); bufUpto += Character.charCount(codePoint); } @@ -407,7 +407,7 @@ public final class SynonymFilter extends TokenFilter { // OK, entire token matched; now see if this is a final // state: if (scratchArc.isFinal()) { - matchOutput = fst.outputs.add(pendingOutput, scratchArc.nextFinalOutput); + matchOutput = fst.outputs.add(pendingOutput, scratchArc.nextFinalOutput()); matchInputLength = tokenCount; matchEndOffset = inputEndOffset; //System.out.println(" found matchLength=" + matchInputLength + " output=" + matchOutput); @@ -423,7 +423,7 @@ public final class SynonymFilter extends TokenFilter { } else { // More matching is possible -- accum the output (if // any) of the WORD_SEP arc: - pendingOutput = fst.outputs.add(pendingOutput, scratchArc.output); + pendingOutput = fst.outputs.add(pendingOutput, scratchArc.output()); if (nextRead == nextWrite) { capture(); } diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymGraphFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymGraphFilter.java index e59e61bf723..c6cbea85323 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymGraphFilter.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymGraphFilter.java @@ -290,7 +290,7 @@ public final class SynonymGraphFilter extends TokenFilter { BytesRef pendingOutput = fst.outputs.getNoOutput(); fst.getFirstArc(scratchArc); - assert scratchArc.output == fst.outputs.getNoOutput(); + assert scratchArc.output() == fst.outputs.getNoOutput(); // How many tokens in the current match int matchLength = 0; @@ -360,7 +360,7 @@ public final class SynonymGraphFilter extends TokenFilter { } // Accum the output - pendingOutput = fst.outputs.add(pendingOutput, scratchArc.output); + pendingOutput = fst.outputs.add(pendingOutput, scratchArc.output()); bufUpto += Character.charCount(codePoint); } @@ -369,7 +369,7 @@ public final class SynonymGraphFilter extends TokenFilter { // OK, entire token matched; now see if this is a final // state in the FST (a match): if (scratchArc.isFinal()) { - matchOutput = fst.outputs.add(pendingOutput, scratchArc.nextFinalOutput); + matchOutput = fst.outputs.add(pendingOutput, scratchArc.nextFinalOutput()); matchInputLength = matchLength; matchEndOffset = inputEndOffset; //System.out.println(" ** match"); @@ -385,7 +385,7 @@ public final class SynonymGraphFilter extends TokenFilter { } else { // More matching is possible -- accum the output (if // any) of the WORD_SEP arc: - pendingOutput = fst.outputs.add(pendingOutput, scratchArc.output); + pendingOutput = fst.outputs.add(pendingOutput, scratchArc.output()); doFinalCapture = true; if (liveToken) { capture(); diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java index ea57d1c28c8..96d104bde1c 100644 --- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java +++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java @@ -772,12 +772,12 @@ public final class JapaneseTokenizer extends Tokenizer { if (userFST.findTargetArc(ch, arc, arc, posAhead == posData.pos, userFSTReader) == null) { break; } - output += arc.output.intValue(); + output += arc.output().intValue(); if (arc.isFinal()) { if (VERBOSE) { System.out.println(" USER word " + new String(buffer.get(pos, posAhead - pos + 1)) + " toPos=" + (posAhead + 1)); } - add(userDictionary, posData, posAhead+1, output + arc.nextFinalOutput.intValue(), Type.USER, false); + add(userDictionary, posData, posAhead+1, output + arc.nextFinalOutput().intValue(), Type.USER, false); anyMatches = true; } } @@ -803,7 +803,7 @@ public final class JapaneseTokenizer extends Tokenizer { break; } - output += arc.output.intValue(); + output += arc.output().intValue(); // Optimization: for known words that are too-long // (compound), we should pre-compute the 2nd @@ -812,7 +812,7 @@ public final class JapaneseTokenizer extends Tokenizer { // match is found. if (arc.isFinal()) { - dictionary.lookupWordIds(output + arc.nextFinalOutput.intValue(), wordIdRef); + dictionary.lookupWordIds(output + arc.nextFinalOutput().intValue(), wordIdRef); if (VERBOSE) { System.out.println(" KNOWN word " + new String(buffer.get(pos, posAhead - pos + 1)) + " toPos=" + (posAhead + 1) + " " + wordIdRef.length + " wordIDs"); } diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java index bf2ef57f3d6..eaa5badd17a 100644 --- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java +++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java @@ -162,9 +162,9 @@ public final class UserDictionary implements Dictionary { if (fst.findTargetArc(ch, arc, arc, i == 0, fstReader) == null) { break; // continue to next position } - output += arc.output.intValue(); + output += arc.output().intValue(); if (arc.isFinal()) { - final int finalOutput = output + arc.nextFinalOutput.intValue(); + final int finalOutput = output + arc.nextFinalOutput().intValue(); result.put(startOffset-off, segmentations[finalOutput]); found = true; } diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java index 000c743842c..300740095cc 100644 --- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java +++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java @@ -681,11 +681,11 @@ public final class KoreanTokenizer extends Tokenizer { if (userFST.findTargetArc(ch, arc, arc, posAhead == pos, userFSTReader) == null) { break; } - output += arc.output.intValue(); + output += arc.output().intValue(); if (arc.isFinal()) { maxPosAhead = posAhead; outputMaxPosAhead = output; - arcFinalOutMaxPosAhead = arc.nextFinalOutput.intValue(); + arcFinalOutMaxPosAhead = arc.nextFinalOutput().intValue(); anyMatches = true; } } @@ -720,7 +720,7 @@ public final class KoreanTokenizer extends Tokenizer { break; } - output += arc.output.intValue(); + output += arc.output().intValue(); // Optimization: for known words that are too-long // (compound), we should pre-compute the 2nd @@ -729,7 +729,7 @@ public final class KoreanTokenizer extends Tokenizer { // match is found. if (arc.isFinal()) { - dictionary.lookupWordIds(output + arc.nextFinalOutput.intValue(), wordIdRef); + dictionary.lookupWordIds(output + arc.nextFinalOutput().intValue(), wordIdRef); if (VERBOSE) { System.out.println(" KNOWN word " + new String(buffer.get(pos, posAhead - pos + 1)) + " toPos=" + (posAhead + 1) + " " + wordIdRef.length + " wordIDs"); } diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java index fb601be6d65..e04d133335e 100644 --- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java +++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java @@ -221,9 +221,9 @@ public final class UserDictionary implements Dictionary { if (fst.findTargetArc(ch, arc, arc, i == 0, fstReader) == null) { break; // continue to next position } - output += arc.output.intValue(); + output += arc.output().intValue(); if (arc.isFinal()) { - final int finalOutput = output + arc.nextFinalOutput.intValue(); + final int finalOutput = output + arc.nextFinalOutput().intValue(); result.add(finalOutput); } } diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java index 9434ca8928d..fc7984d5d55 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java @@ -95,7 +95,7 @@ final class OrdsIntersectTermsEnum extends BaseTermsEnum { f.prefix = 0; f.setState(0); f.arc = arc; - f.outputPrefix = arc.output; + f.outputPrefix = arc.output(); f.load(fr.rootCode); // for assert: @@ -168,14 +168,14 @@ final class OrdsIntersectTermsEnum extends BaseTermsEnum { // passed to findTargetArc arc = fr.index.findTargetArc(target, arc, getArc(1+idx), fstReader); assert arc != null; - output = OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output); + output = OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output()); idx++; } f.arc = arc; f.outputPrefix = output; assert arc.isFinal(); - f.load(OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput)); + f.load(OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput())); return f; } diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java index e03d7e672d1..7bfaab55ffa 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java @@ -271,7 +271,7 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { arc = arcs[0]; assert arc.isFinal(); - output = arc.output; + output = arc.output(); targetUpto = 0; OrdsSegmentTermsEnumFrame lastFrame = stack[0]; @@ -294,9 +294,9 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { break; } arc = arcs[1+targetUpto]; - assert arc.label == (target.bytes[target.offset + targetUpto] & 0xFF): "arc.label=" + (char) arc.label + " targetLabel=" + (char) (target.bytes[target.offset + targetUpto] & 0xFF); - if (arc.output != OrdsBlockTreeTermsWriter.NO_OUTPUT) { - output = OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output); + assert arc.label() == (target.bytes[target.offset + targetUpto] & 0xFF): "arc.label=" + (char) arc.label() + " targetLabel=" + (char) (target.bytes[target.offset + targetUpto] & 0xFF); + if (arc.output() != OrdsBlockTreeTermsWriter.NO_OUTPUT) { + output = OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output()); } if (arc.isFinal()) { lastFrame = stack[1+lastFrame.ord]; @@ -374,19 +374,19 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { // Empty string prefix must have an output (block) in the index! assert arc.isFinal(); - assert arc.output != null; + assert arc.output() != null; // if (DEBUG) { // System.out.println(" no seek state; push root frame"); // } - output = arc.output; + output = arc.output(); currentFrame = staticFrame; //term.length = 0; targetUpto = 0; - currentFrame = pushFrame(arc, OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput), 0); + currentFrame = pushFrame(arc, OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput()), 0); } positioned = true; @@ -443,9 +443,9 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { arc = nextArc; term.setByteAt(targetUpto, (byte) targetLabel); // Aggregate output as we go: - assert arc.output != null; - if (arc.output != OrdsBlockTreeTermsWriter.NO_OUTPUT) { - output = OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output); + assert arc.output() != null; + if (arc.output() != OrdsBlockTreeTermsWriter.NO_OUTPUT) { + output = OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output()); } // if (DEBUG) { @@ -455,7 +455,7 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { if (arc.isFinal()) { //if (DEBUG) System.out.println(" arc is final!"); - currentFrame = pushFrame(arc, OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput), targetUpto); + currentFrame = pushFrame(arc, OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput()), targetUpto); //if (DEBUG) System.out.println(" curFrame.ord=" + currentFrame.ord + " hasTerms=" + currentFrame.hasTerms); } } @@ -529,7 +529,7 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { arc = arcs[0]; assert arc.isFinal(); - output = arc.output; + output = arc.output(); targetUpto = 0; OrdsSegmentTermsEnumFrame lastFrame = stack[0]; @@ -552,14 +552,14 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { break; } arc = arcs[1+targetUpto]; - assert arc.label == (target.bytes[target.offset + targetUpto] & 0xFF): "arc.label=" + (char) arc.label + " targetLabel=" + (char) (target.bytes[target.offset + targetUpto] & 0xFF); + assert arc.label() == (target.bytes[target.offset + targetUpto] & 0xFF): "arc.label=" + (char) arc.label() + " targetLabel=" + (char) (target.bytes[target.offset + targetUpto] & 0xFF); // TODO: we could save the outputs in local // byte[][] instead of making new objs ever // seek; but, often the FST doesn't have any // shared bytes (but this could change if we // reverse vLong byte order) - if (arc.output != OrdsBlockTreeTermsWriter.NO_OUTPUT) { - output = OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output); + if (arc.output() != OrdsBlockTreeTermsWriter.NO_OUTPUT) { + output = OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output()); } if (arc.isFinal()) { lastFrame = stack[1+lastFrame.ord]; @@ -632,19 +632,19 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { // Empty string prefix must have an output (block) in the index! assert arc.isFinal(); - assert arc.output != null; + assert arc.output() != null; //if (DEBUG) { //System.out.println(" no seek state; push root frame"); //} - output = arc.output; + output = arc.output(); currentFrame = staticFrame; //term.length = 0; targetUpto = 0; - currentFrame = pushFrame(arc, OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput), 0); + currentFrame = pushFrame(arc, OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput()), 0); } positioned = true; @@ -701,9 +701,9 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { term.setByteAt(targetUpto, (byte) targetLabel); arc = nextArc; // Aggregate output as we go: - assert arc.output != null; - if (arc.output != OrdsBlockTreeTermsWriter.NO_OUTPUT) { - output = OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output); + assert arc.output() != null; + if (arc.output() != OrdsBlockTreeTermsWriter.NO_OUTPUT) { + output = OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output()); } //if (DEBUG) { @@ -713,7 +713,7 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { if (arc.isFinal()) { //if (DEBUG) System.out.println(" arc is final!"); - currentFrame = pushFrame(arc, OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput), targetUpto); + currentFrame = pushFrame(arc, OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput()), targetUpto); //if (DEBUG) System.out.println(" curFrame.ord=" + currentFrame.ord + " hasTerms=" + currentFrame.hasTerms); } } @@ -766,8 +766,8 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { } if (fr.index != null) { assert !isSeekFrame || f.arc != null: "isSeekFrame=" + isSeekFrame + " f.arc=" + f.arc; - if (f.prefix > 0 && isSeekFrame && f.arc.label != (term.byteAt(f.prefix-1)&0xFF)) { - out.println(" broken seek state: arc.label=" + (char) f.arc.label + " vs term byte=" + (char) (term.byteAt(f.prefix-1)&0xFF)); + if (f.prefix > 0 && isSeekFrame && f.arc.label() != (term.byteAt(f.prefix-1)&0xFF)) { + out.println(" broken seek state: arc.label=" + (char) f.arc.label() + " vs term byte=" + (char) (term.byteAt(f.prefix-1)&0xFF)); throw new RuntimeException("seek state is broken"); } Output output = Util.get(fr.index, prefix); @@ -1052,7 +1052,7 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { final IntsRefBuilder result = new IntsRefBuilder(); fr.index.getFirstArc(arc); - Output output = arc.output; + Output output = arc.output(); int upto = 0; int bestUpto = 0; @@ -1069,7 +1069,7 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { while (true) { // System.out.println(" loop: output=" + output.startOrd + "-" + (Long.MAX_VALUE-output.endOrd) + " upto=" + upto + " arc=" + arc + " final?=" + arc.isFinal()); if (arc.isFinal()) { - final Output finalOutput = OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput); + final Output finalOutput = OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput()); // System.out.println(" isFinal: " + finalOutput.startOrd + "-" + (Long.MAX_VALUE-finalOutput.endOrd)); if (targetOrd >= finalOutput.startOrd && targetOrd <= Long.MAX_VALUE-finalOutput.endOrd) { // Only one range should match across all arc leaving this node @@ -1082,19 +1082,19 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { if (FST.targetHasArcs(arc)) { // System.out.println(" targetHasArcs"); result.grow(1+upto); - fr.index.readFirstRealTargetArc(arc.target, arc, fstReader); + fr.index.readFirstRealTargetArc(arc.target(), arc, fstReader); - if (arc.bytesPerArc != 0 && arc.arcIdx > Integer.MIN_VALUE) { + if (arc.bytesPerArc() != 0 && arc.arcIdx() > Integer.MIN_VALUE) { // System.out.println(" array arcs"); int low = 0; - int high = arc.numArcs-1; + int high = arc.numArcs() -1; int mid = 0; //System.out.println("bsearch: numArcs=" + arc.numArcs + " target=" + targetOutput + " output=" + output); boolean found = false; while (low <= high) { mid = (low + high) >>> 1; - fstReader.setPosition(arc.posArcsStart); - fstReader.skipBytes(arc.bytesPerArc*mid); + fstReader.setPosition(arc.posArcsStart()); + fstReader.skipBytes(arc.bytesPerArc() *mid); final byte flags = fstReader.readByte(); fr.index.readLabel(fstReader); final Output minArcOutput; @@ -1116,8 +1116,8 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { } if (found) { - // Keep recursing - arc.arcIdx = mid-1; + // Keep recursing + arc.arcIdx(mid - 1); } else { result.setLength(bestUpto); InputOutput io = new InputOutput(); @@ -1130,8 +1130,8 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { fr.index.readNextRealArc(arc, fstReader); // Recurse on this arc: - result.setIntAt(upto++, arc.label); - output = OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output); + result.setIntAt(upto++, arc.label()); + output = OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output()); } else { // System.out.println(" non-array arc"); @@ -1141,14 +1141,14 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { // This is the min output we'd hit if we follow // this arc: - final Output minArcOutput = OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output); + final Output minArcOutput = OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output()); long endOrd = Long.MAX_VALUE - minArcOutput.endOrd; // System.out.println(" endOrd=" + endOrd + " targetOrd=" + targetOrd); if (targetOrd >= minArcOutput.startOrd && targetOrd <= endOrd) { // Recurse on this arc: output = minArcOutput; - result.setIntAt(upto++, arc.label); + result.setIntAt(upto++, arc.label()); break; } else if (targetOrd < endOrd || arc.isLast()) { result.setLength(bestUpto); diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java index 12110d9a811..d653c1209dd 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java @@ -621,8 +621,8 @@ public class FSTOrdTermsReader extends FieldsProducer { @Override void decodeStats() throws IOException { final FST.Arc arc = topFrame().arc; - assert arc.nextFinalOutput == fstOutputs.getNoOutput(); - ord = arc.output; + assert arc.nextFinalOutput() == fstOutputs.getNoOutput(); + ord = arc.output(); super.decodeStats(); } @@ -675,7 +675,7 @@ public class FSTOrdTermsReader extends FieldsProducer { frame = newFrame(); label = target.bytes[upto] & 0xff; frame = loadCeilFrame(label, topFrame(), frame); - if (frame == null || frame.arc.label != label) { + if (frame == null || frame.arc.label() != label) { break; } assert isValid(frame); // target must be fetched from automaton @@ -703,15 +703,15 @@ public class FSTOrdTermsReader extends FieldsProducer { } /** Virtual frame, never pop */ - Frame loadVirtualFrame(Frame frame) throws IOException { - frame.arc.output = fstOutputs.getNoOutput(); - frame.arc.nextFinalOutput = fstOutputs.getNoOutput(); + Frame loadVirtualFrame(Frame frame) { + frame.arc.output(fstOutputs.getNoOutput()); + frame.arc.nextFinalOutput(fstOutputs.getNoOutput()); frame.state = -1; return frame; } /** Load frame for start arc(node) on fst */ - Frame loadFirstFrame(Frame frame) throws IOException { + Frame loadFirstFrame(Frame frame) { frame.arc = fst.getFirstArc(frame.arc); frame.state = 0; return frame; @@ -722,8 +722,8 @@ public class FSTOrdTermsReader extends FieldsProducer { if (!canGrow(top)) { return null; } - frame.arc = fst.readFirstRealTargetArc(top.arc.target, frame.arc, fstReader); - frame.state = fsa.step(top.state, frame.arc.label); + frame.arc = fst.readFirstRealTargetArc(top.arc.target(), frame.arc, fstReader); + frame.state = fsa.step(top.state, frame.arc.label()); //if (TEST) System.out.println(" loadExpand frame="+frame); if (frame.state == -1) { return loadNextFrame(top, frame); @@ -738,7 +738,7 @@ public class FSTOrdTermsReader extends FieldsProducer { } while (!frame.arc.isLast()) { frame.arc = fst.readNextRealArc(frame.arc, fstReader); - frame.state = fsa.step(top.state, frame.arc.label); + frame.state = fsa.step(top.state, frame.arc.label()); if (frame.state != -1) { break; } @@ -758,7 +758,7 @@ public class FSTOrdTermsReader extends FieldsProducer { if (arc == null) { return null; } - frame.state = fsa.step(top.state, arc.label); + frame.state = fsa.step(top.state, arc.label()); //if (TEST) System.out.println(" loadCeil frame="+frame); if (frame.state == -1) { return loadNextFrame(top, frame); @@ -781,8 +781,8 @@ public class FSTOrdTermsReader extends FieldsProducer { void pushFrame(Frame frame) { final FST.Arc arc = frame.arc; - arc.output = fstOutputs.add(topFrame().arc.output, arc.output); - term = grow(arc.label); + arc.output(fstOutputs.add(topFrame().arc.output(), arc.output())); + term = grow(arc.label()); level++; assert frame == stack[level]; } @@ -836,7 +836,7 @@ public class FSTOrdTermsReader extends FieldsProducer { queue.add(startArc); while (!queue.isEmpty()) { final FST.Arc arc = queue.remove(0); - final long node = arc.target; + final long node = arc.target(); //System.out.println(arc); if (FST.targetHasArcs(arc) && !seen.get((int) node)) { seen.set((int) node); diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java index 43528ced1f0..41a992fafea 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java @@ -415,7 +415,7 @@ public class FSTTermsReader extends FieldsProducer { /* True when there is pending term when calling next() */ boolean pending; - /* stack to record how current term is constructed, + /* stack to record how current term is constructed, * used to accumulate metadata or rewind term: * level == term.length + 1, * == 0 when term is null */ @@ -501,19 +501,19 @@ public class FSTTermsReader extends FieldsProducer { } /** Lazily accumulate meta data, when we got a accepted term */ - void loadMetaData() throws IOException { + void loadMetaData() { FST.Arc last, next; last = stack[metaUpto].fstArc; while (metaUpto != level) { metaUpto++; next = stack[metaUpto].fstArc; - next.output = fstOutputs.add(next.output, last.output); + next.output(fstOutputs.add(next.output(), last.output())); last = next; } if (last.isFinal()) { - meta = fstOutputs.add(last.output, last.nextFinalOutput); + meta = fstOutputs.add(last.output(), last.nextFinalOutput()); } else { - meta = last.output; + meta = last.output(); } state.docFreq = meta.docFreq; state.totalTermFreq = meta.totalTermFreq; @@ -575,7 +575,7 @@ public class FSTTermsReader extends FieldsProducer { frame = newFrame(); label = target.bytes[upto] & 0xff; frame = loadCeilFrame(label, topFrame(), frame); - if (frame == null || frame.fstArc.label != label) { + if (frame == null || frame.fstArc.label() != label) { break; } assert isValid(frame); // target must be fetched from automaton @@ -603,9 +603,9 @@ public class FSTTermsReader extends FieldsProducer { } /** Virtual frame, never pop */ - Frame loadVirtualFrame(Frame frame) throws IOException { - frame.fstArc.output = fstOutputs.getNoOutput(); - frame.fstArc.nextFinalOutput = fstOutputs.getNoOutput(); + Frame loadVirtualFrame(Frame frame) { + frame.fstArc.output(fstOutputs.getNoOutput()); + frame.fstArc.nextFinalOutput(fstOutputs.getNoOutput()); frame.fsaState = -1; return frame; } @@ -622,8 +622,8 @@ public class FSTTermsReader extends FieldsProducer { if (!canGrow(top)) { return null; } - frame.fstArc = fst.readFirstRealTargetArc(top.fstArc.target, frame.fstArc, fstReader); - frame.fsaState = fsa.step(top.fsaState, frame.fstArc.label); + frame.fstArc = fst.readFirstRealTargetArc(top.fstArc.target(), frame.fstArc, fstReader); + frame.fsaState = fsa.step(top.fsaState, frame.fstArc.label()); //if (TEST) System.out.println(" loadExpand frame="+frame); if (frame.fsaState == -1) { return loadNextFrame(top, frame); @@ -638,7 +638,7 @@ public class FSTTermsReader extends FieldsProducer { } while (!frame.fstArc.isLast()) { frame.fstArc = fst.readNextRealArc(frame.fstArc, fstReader); - frame.fsaState = fsa.step(top.fsaState, frame.fstArc.label); + frame.fsaState = fsa.step(top.fsaState, frame.fstArc.label()); if (frame.fsaState != -1) { break; } @@ -658,7 +658,7 @@ public class FSTTermsReader extends FieldsProducer { if (arc == null) { return null; } - frame.fsaState = fsa.step(top.fsaState, arc.label); + frame.fsaState = fsa.step(top.fsaState, arc.label()); //if (TEST) System.out.println(" loadCeil frame="+frame); if (frame.fsaState == -1) { return loadNextFrame(top, frame); @@ -680,7 +680,7 @@ public class FSTTermsReader extends FieldsProducer { } void pushFrame(Frame frame) { - term = grow(frame.fstArc.label); + term = grow(frame.fstArc.label()); level++; //if (TEST) System.out.println(" term=" + term + " level=" + level); } @@ -737,7 +737,7 @@ public class FSTTermsReader extends FieldsProducer { queue.add(startArc); while (!queue.isEmpty()) { final FST.Arc arc = queue.remove(0); - final long node = arc.target; + final long node = arc.target(); //System.out.println(arc); if (FST.targetHasArcs(arc) && !seen.get((int) node)) { seen.set((int) node); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java index 848bb0b7528..76bfdf23dde 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java @@ -111,7 +111,7 @@ final class IntersectTermsEnum extends BaseTermsEnum { f.prefix = 0; f.setState(0); f.arc = arc; - f.outputPrefix = arc.output; + f.outputPrefix = arc.output(); f.load(fr.rootCode); // for assert: @@ -186,14 +186,14 @@ final class IntersectTermsEnum extends BaseTermsEnum { // passed to findTargetArc arc = fr.index.findTargetArc(target, arc, getArc(1+idx), fstReader); assert arc != null; - output = fstOutputs.add(output, arc.output); + output = fstOutputs.add(output, arc.output()); idx++; } f.arc = arc; f.outputPrefix = output; assert arc.isFinal(); - f.load(fstOutputs.add(output, arc.nextFinalOutput)); + f.load(fstOutputs.add(output, arc.nextFinalOutput())); return f; } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java index c9d0ddf6419..92888d060c9 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java @@ -351,7 +351,7 @@ final class SegmentTermsEnum extends BaseTermsEnum { arc = arcs[0]; assert arc.isFinal(); - output = arc.output; + output = arc.output(); targetUpto = 0; SegmentTermsEnumFrame lastFrame = stack[0]; @@ -374,9 +374,9 @@ final class SegmentTermsEnum extends BaseTermsEnum { break; } arc = arcs[1+targetUpto]; - assert arc.label == (target.bytes[target.offset + targetUpto] & 0xFF): "arc.label=" + (char) arc.label + " targetLabel=" + (char) (target.bytes[target.offset + targetUpto] & 0xFF); - if (arc.output != BlockTreeTermsReader.NO_OUTPUT) { - output = BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.output); + assert arc.label() == (target.bytes[target.offset + targetUpto] & 0xFF): "arc.label=" + (char) arc.label() + " targetLabel=" + (char) (target.bytes[target.offset + targetUpto] & 0xFF); + if (arc.output() != BlockTreeTermsReader.NO_OUTPUT) { + output = BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.output()); } if (arc.isFinal()) { lastFrame = stack[1+lastFrame.ord]; @@ -454,19 +454,19 @@ final class SegmentTermsEnum extends BaseTermsEnum { // Empty string prefix must have an output (block) in the index! assert arc.isFinal(); - assert arc.output != null; + assert arc.output() != null; // if (DEBUG) { // System.out.println(" no seek state; push root frame"); // } - output = arc.output; + output = arc.output(); currentFrame = staticFrame; //term.length = 0; targetUpto = 0; - currentFrame = pushFrame(arc, BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.nextFinalOutput), 0); + currentFrame = pushFrame(arc, BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.nextFinalOutput()), 0); } // if (DEBUG) { @@ -521,9 +521,9 @@ final class SegmentTermsEnum extends BaseTermsEnum { arc = nextArc; term.setByteAt(targetUpto, (byte) targetLabel); // Aggregate output as we go: - assert arc.output != null; - if (arc.output != BlockTreeTermsReader.NO_OUTPUT) { - output = BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.output); + assert arc.output() != null; + if (arc.output() != BlockTreeTermsReader.NO_OUTPUT) { + output = BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.output()); } // if (DEBUG) { @@ -533,7 +533,7 @@ final class SegmentTermsEnum extends BaseTermsEnum { if (arc.isFinal()) { //if (DEBUG) System.out.println(" arc is final!"); - currentFrame = pushFrame(arc, BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.nextFinalOutput), targetUpto); + currentFrame = pushFrame(arc, BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.nextFinalOutput()), targetUpto); //if (DEBUG) System.out.println(" curFrame.ord=" + currentFrame.ord + " hasTerms=" + currentFrame.hasTerms); } } @@ -608,7 +608,7 @@ final class SegmentTermsEnum extends BaseTermsEnum { arc = arcs[0]; assert arc.isFinal(); - output = arc.output; + output = arc.output(); targetUpto = 0; SegmentTermsEnumFrame lastFrame = stack[0]; @@ -631,14 +631,14 @@ final class SegmentTermsEnum extends BaseTermsEnum { break; } arc = arcs[1+targetUpto]; - assert arc.label == (target.bytes[target.offset + targetUpto] & 0xFF): "arc.label=" + (char) arc.label + " targetLabel=" + (char) (target.bytes[target.offset + targetUpto] & 0xFF); + assert arc.label() == (target.bytes[target.offset + targetUpto] & 0xFF): "arc.label=" + (char) arc.label() + " targetLabel=" + (char) (target.bytes[target.offset + targetUpto] & 0xFF); // TODO: we could save the outputs in local // byte[][] instead of making new objs ever // seek; but, often the FST doesn't have any // shared bytes (but this could change if we // reverse vLong byte order) - if (arc.output != BlockTreeTermsReader.NO_OUTPUT) { - output = BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.output); + if (arc.output() != BlockTreeTermsReader.NO_OUTPUT) { + output = BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.output()); } if (arc.isFinal()) { lastFrame = stack[1+lastFrame.ord]; @@ -711,19 +711,19 @@ final class SegmentTermsEnum extends BaseTermsEnum { // Empty string prefix must have an output (block) in the index! assert arc.isFinal(); - assert arc.output != null; + assert arc.output() != null; //if (DEBUG) { //System.out.println(" no seek state; push root frame"); //} - output = arc.output; + output = arc.output(); currentFrame = staticFrame; //term.length = 0; targetUpto = 0; - currentFrame = pushFrame(arc, BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.nextFinalOutput), 0); + currentFrame = pushFrame(arc, BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.nextFinalOutput()), 0); } //if (DEBUG) { @@ -779,9 +779,9 @@ final class SegmentTermsEnum extends BaseTermsEnum { term.setByteAt(targetUpto, (byte) targetLabel); arc = nextArc; // Aggregate output as we go: - assert arc.output != null; - if (arc.output != BlockTreeTermsReader.NO_OUTPUT) { - output = BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.output); + assert arc.output() != null; + if (arc.output() != BlockTreeTermsReader.NO_OUTPUT) { + output = BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.output()); } //if (DEBUG) { @@ -791,7 +791,7 @@ final class SegmentTermsEnum extends BaseTermsEnum { if (arc.isFinal()) { //if (DEBUG) System.out.println(" arc is final!"); - currentFrame = pushFrame(arc, BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.nextFinalOutput), targetUpto); + currentFrame = pushFrame(arc, BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.nextFinalOutput()), targetUpto); //if (DEBUG) System.out.println(" curFrame.ord=" + currentFrame.ord + " hasTerms=" + currentFrame.hasTerms); } } @@ -844,8 +844,8 @@ final class SegmentTermsEnum extends BaseTermsEnum { } if (fr.index != null) { assert !isSeekFrame || f.arc != null: "isSeekFrame=" + isSeekFrame + " f.arc=" + f.arc; - if (f.prefix > 0 && isSeekFrame && f.arc.label != (term.byteAt(f.prefix-1)&0xFF)) { - out.println(" broken seek state: arc.label=" + (char) f.arc.label + " vs term byte=" + (char) (term.byteAt(f.prefix-1)&0xFF)); + if (f.prefix > 0 && isSeekFrame && f.arc.label() != (term.byteAt(f.prefix-1)&0xFF)) { + out.println(" broken seek state: arc.label=" + (char) f.arc.label() + " vs term byte=" + (char) (term.byteAt(f.prefix-1)&0xFF)); throw new RuntimeException("seek state is broken"); } BytesRef output = Util.get(fr.index, prefix); diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java index f308f1aae65..f9d0cd51605 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java @@ -62,24 +62,24 @@ import org.apache.lucene.util.RamUsageEstimator; */ public final class FST implements Accountable { + /** Specifies allowed range of each int input label for + * this FST. */ + public enum INPUT_TYPE {BYTE1, BYTE2, BYTE4} + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FST.class); private static final long ARC_SHALLOW_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Arc.class); - /** Specifies allowed range of each int input label for - * this FST. */ - public static enum INPUT_TYPE {BYTE1, BYTE2, BYTE4}; - - static final int BIT_FINAL_ARC = 1 << 0; + private static final int BIT_FINAL_ARC = 1 << 0; static final int BIT_LAST_ARC = 1 << 1; static final int BIT_TARGET_NEXT = 1 << 2; // TODO: we can free up a bit if we can nuke this: - static final int BIT_STOP_NODE = 1 << 3; + private static final int BIT_STOP_NODE = 1 << 3; /** This flag is set if the arc has an output. */ public static final int BIT_ARC_HAS_OUTPUT = 1 << 4; - static final int BIT_ARC_HAS_FINAL_OUTPUT = 1 << 5; + private static final int BIT_ARC_HAS_FINAL_OUTPUT = 1 << 5; // We use this as a marker (because this one flag is // illegal by itself ...): @@ -119,10 +119,13 @@ public final class FST implements Accountable { // non-final node w/ no arcs: private static final long NON_FINAL_END_NODE = 0; + /* Used for memory accounting */ + private int cachedArcsBytesUsed; + /** If arc has this label then that arc is final/accepted */ public static final int END_LABEL = -1; - public final INPUT_TYPE inputType; + final INPUT_TYPE inputType; // if non-null, this FST accepts the empty string and // produces this output @@ -139,62 +142,51 @@ public final class FST implements Accountable { public final Outputs outputs; - private Arc cachedRootArcs[]; + private Arc[] cachedRootArcs; /** Represents a single arc. */ public static final class Arc { - public int label; - public T output; - /** To node (ord or address) */ - public long target; + private int label; - byte flags; - public T nextFinalOutput; + private T output; + + private long target; + + private byte flags; + + private T nextFinalOutput; // address (into the byte[]), or ord/address if label == END_LABEL - long nextArc; + private long nextArc; - /** Where the first arc in the array starts; only valid if - * bytesPerArc != 0 */ - public long posArcsStart; - - /** Non-zero if this arc is part of an array, which means all - * arcs for the node are encoded with a fixed number of bytes so - * that we can random access by index. We do when there are enough - * arcs leaving one node. It wastes some bytes but gives faster - * lookups. */ - public int bytesPerArc; + private long posArcsStart; - /** Where we are in the array; only valid if bytesPerArc != 0, and the array has no holes. - * arcIdx = Integer.MIN_VALUE indicates that the arc is part of a direct array, addressed by - * label. - */ - public int arcIdx; + private int bytesPerArc; - /** How many arc, if bytesPerArc == 0. Otherwise, the size of the arc array. If the array is - * direct, this may include holes. Otherwise it is also how many arcs are in the array */ - public int numArcs; + private int arcIdx; + + private int numArcs; /** Returns this */ public Arc copyFrom(Arc other) { - label = other.label; - target = other.target; - flags = other.flags; - output = other.output; - nextFinalOutput = other.nextFinalOutput; - nextArc = other.nextArc; - bytesPerArc = other.bytesPerArc; - if (bytesPerArc != 0) { - posArcsStart = other.posArcsStart; - arcIdx = other.arcIdx; - numArcs = other.numArcs; + label = other.label(); + target = other.target(); + flags = other.flags(); + output = other.output(); + nextFinalOutput = other.nextFinalOutput(); + nextArc = other.nextArc(); + bytesPerArc = other.bytesPerArc(); + if (bytesPerArc() != 0) { + posArcsStart = other.posArcsStart(); + arcIdx = other.arcIdx(); + numArcs = other.numArcs(); } return this; } boolean flag(int flag) { - return FST.flag(flags, flag); + return FST.flag(flags(), flag); } public boolean isLast() { @@ -208,8 +200,8 @@ public final class FST implements Accountable { @Override public String toString() { StringBuilder b = new StringBuilder(); - b.append(" target=").append(target); - b.append(" label=0x").append(Integer.toHexString(label)); + b.append(" target=").append(target()); + b.append(" label=0x").append(Integer.toHexString(label())); if (flag(BIT_FINAL_ARC)) { b.append(" final"); } @@ -223,40 +215,121 @@ public final class FST implements Accountable { b.append(" stop"); } if (flag(BIT_ARC_HAS_OUTPUT)) { - b.append(" output=").append(output); + b.append(" output=").append(output()); } if (flag(BIT_ARC_HAS_FINAL_OUTPUT)) { - b.append(" nextFinalOutput=").append(nextFinalOutput); + b.append(" nextFinalOutput=").append(nextFinalOutput()); } - if (bytesPerArc != 0) { - b.append(" arcArray(idx=").append(arcIdx).append(" of ").append(numArcs).append(")"); + if (bytesPerArc() != 0) { + b.append(" arcArray(idx=").append(arcIdx()).append(" of ").append(numArcs()).append(")"); } return b.toString(); } - }; + + public int label() { + return label; + } + + public void label(int label) { + this.label = label; + } + + public T output() { + return output; + } + + public void output(T output) { + this.output = output; + } + + /** To node (ord or address) */ + public long target() { + return target; + } + + public byte flags() { + return flags; + } + + public void flags(byte flags) { + this.flags = flags; + } + + public T nextFinalOutput() { + return nextFinalOutput; + } + + public void nextFinalOutput(T output) { + nextFinalOutput = output; + } + + long nextArc() { + return nextArc; + } + + /** + * Set the position of the next arc to read + * @param nextArc the position to set + */ + public void nextArc(long nextArc) { + this.nextArc = nextArc; + } + + /** Where the first arc in the array starts; only valid if + * bytesPerArc != 0 */ + public long posArcsStart() { + return posArcsStart; + } + + /** Non-zero if this arc is part of an array, which means all + * arcs for the node are encoded with a fixed number of bytes so + * that we can random access by index. We do when there are enough + * arcs leaving one node. It wastes some bytes but gives faster + * lookups. */ + public int bytesPerArc() { + return bytesPerArc; + } + + /** Where we are in the array; only valid if bytesPerArc != 0, and the array has no holes. + * arcIdx = Integer.MIN_VALUE indicates that the arc is part of a direct array, addressed by + * label. + */ + public int arcIdx() { + return arcIdx; + } + + /** + * Set the arcIdx + * @param idx the value to set + */ + public void arcIdx(int idx) { + arcIdx = idx; + } + + /** How many arc, if bytesPerArc == 0. Otherwise, the size of the arc array. If the array is + * direct, this may include holes. Otherwise it is also how many arcs are in the array */ + public int numArcs() { + return numArcs; + } + } private static boolean flag(int flags, int bit) { return (flags & bit) != 0; } - private final int version; - - // make a new empty FST, for building; Builder invokes - // this ctor + // make a new empty FST, for building; Builder invokes this FST(INPUT_TYPE inputType, Outputs outputs, int bytesPageBits) { this.inputType = inputType; this.outputs = outputs; - version = VERSION_CURRENT; fstStore = null; bytes = new BytesStore(bytesPageBits); // pad: ensure no node gets address 0 which is reserved to mean // the stop state w/ no arcs bytes.writeByte((byte) 0); - emptyOutput = null; } - public static final int DEFAULT_MAX_BLOCK_BITS = Constants.JRE_IS_64BIT ? 30 : 28; + private static final int DEFAULT_MAX_BLOCK_BITS = Constants.JRE_IS_64BIT ? 30 : 28; /** Load a previously saved FST. */ public FST(DataInput in, Outputs outputs) throws IOException { @@ -270,9 +343,9 @@ public final class FST implements Accountable { this.fstStore = fstStore; this.outputs = outputs; - // NOTE: only reads most recent format; we don't have - // back-compat promise for FSTs (they are experimental): - version = CodecUtil.checkHeader(in, FILE_FORMAT_NAME, VERSION_START, VERSION_CURRENT); + // NOTE: only reads formats VERSION_START up to VERSION_CURRENT; we don't have + // back-compat promise for FSTs (they are experimental), but we are sometimes able to offer it + CodecUtil.checkHeader(in, FILE_FORMAT_NAME, VERSION_START, VERSION_CURRENT); if (in.readByte() == 1) { // accepts empty string // 1 KB blocks: @@ -313,10 +386,6 @@ public final class FST implements Accountable { cacheRootArcs(); } - public INPUT_TYPE getInputType() { - return inputType; - } - private long ramBytesUsed(Arc[] arcs) { long size = 0; if (arcs != null) { @@ -324,11 +393,11 @@ public final class FST implements Accountable { for (Arc arc : arcs) { if (arc != null) { size += ARC_SHALLOW_RAM_BYTES_USED; - if (arc.output != null && arc.output != outputs.getNoOutput()) { - size += outputs.ramBytesUsed(arc.output); + if (arc.output() != null && arc.output() != outputs.getNoOutput()) { + size += outputs.ramBytesUsed(arc.output()); } - if (arc.nextFinalOutput != null && arc.nextFinalOutput != outputs.getNoOutput()) { - size += outputs.ramBytesUsed(arc.nextFinalOutput); + if (arc.nextFinalOutput() != null && arc.nextFinalOutput() != outputs.getNoOutput()) { + size += outputs.ramBytesUsed(arc.nextFinalOutput()); } } } @@ -336,8 +405,6 @@ public final class FST implements Accountable { return size; } - private int cachedArcsBytesUsed; - @Override public long ramBytesUsed() { long size = BASE_RAM_BYTES_USED; @@ -380,12 +447,12 @@ public final class FST implements Accountable { if (targetHasArcs(arc)) { final BytesReader in = getBytesReader(); Arc[] arcs = (Arc[]) new Arc[0x80]; - readFirstRealTargetArc(arc.target, arc, in); + readFirstRealTargetArc(arc.target(), arc, in); int count = 0; while(true) { - assert arc.label != END_LABEL; - if (arc.label < arcs.length) { - arcs[arc.label] = new Arc().copyFrom(arc); + assert arc.label() != END_LABEL; + if (arc.label() < arcs.length) { + arcs[arc.label()] = new Arc().copyFrom(arc); } else { break; } @@ -410,7 +477,7 @@ public final class FST implements Accountable { return emptyOutput; } - void setEmptyOutput(T v) throws IOException { + void setEmptyOutput(T v) { if (emptyOutput != null) { emptyOutput = outputs.merge(emptyOutput, v); } else { @@ -433,18 +500,19 @@ public final class FST implements Accountable { ByteBuffersDataOutput ros = new ByteBuffersDataOutput(); outputs.writeFinalOutput(emptyOutput, ros); byte[] emptyOutputBytes = ros.toArrayCopy(); + int emptyLen = emptyOutputBytes.length; // reverse - final int stopAt = emptyOutputBytes.length/2; + final int stopAt = emptyLen / 2; int upto = 0; while (upto < stopAt) { final byte b = emptyOutputBytes[upto]; - emptyOutputBytes[upto] = emptyOutputBytes[emptyOutputBytes.length-upto-1]; - emptyOutputBytes[emptyOutputBytes.length-upto-1] = b; + emptyOutputBytes[upto] = emptyOutputBytes[emptyLen - upto - 1]; + emptyOutputBytes[emptyLen - upto - 1] = b; upto++; } - out.writeVInt(emptyOutputBytes.length); - out.writeBytes(emptyOutputBytes, 0, emptyOutputBytes.length); + out.writeVInt(emptyLen); + out.writeBytes(emptyOutputBytes, 0, emptyLen); } else { out.writeByte((byte) 0); } @@ -517,7 +585,7 @@ public final class FST implements Accountable { /** returns true if the node at this address has any * outgoing arcs */ public static boolean targetHasArcs(Arc arc) { - return arc.target > 0; + return arc.target() > 0; } // serializes new node by appending its bytes to the end @@ -652,7 +720,7 @@ public final class FST implements Accountable { //System.out.println("write int @pos=" + (fixedArrayStart-4) + " numArcs=" + nodeIn.numArcs); // create the header // TODO: clean this up: or just rewind+reuse and deal with it - byte header[] = new byte[MAX_HEADER_SIZE]; + byte[] header = new byte[MAX_HEADER_SIZE]; ByteArrayDataOutput bad = new ByteArrayDataOutput(header); // write a "false" first arc: if (writeDirectly) { @@ -742,8 +810,7 @@ public final class FST implements Accountable { } } - /** Fills virtual 'start' arc, ie, an empty incoming arc to - * the FST's start node */ + /** Fills virtual 'start' arc, ie, an empty incoming arc to the FST's start node */ public Arc getFirstArc(Arc arc) { T NO_OUTPUT = outputs.getNoOutput(); @@ -771,18 +838,18 @@ public final class FST implements Accountable { * * @return Returns the second argument * (arc). */ - public Arc readLastTargetArc(Arc follow, Arc arc, BytesReader in) throws IOException { + Arc readLastTargetArc(Arc follow, Arc arc, BytesReader in) throws IOException { //System.out.println("readLast"); if (!targetHasArcs(follow)) { //System.out.println(" end node"); assert follow.isFinal(); arc.label = END_LABEL; arc.target = FINAL_END_NODE; - arc.output = follow.nextFinalOutput; + arc.output = follow.nextFinalOutput(); arc.flags = BIT_LAST_ARC; return arc; } else { - in.setPosition(follow.target); + in.setPosition(follow.target()); final byte b = in.readByte(); if (b == ARCS_AS_ARRAY_PACKED || b == ARCS_AS_ARRAY_WITH_GAPS) { // array: jump straight to end @@ -792,9 +859,9 @@ public final class FST implements Accountable { arc.posArcsStart = in.getPosition(); if (b == ARCS_AS_ARRAY_WITH_GAPS) { arc.arcIdx = Integer.MIN_VALUE; - arc.nextArc = arc.posArcsStart - (arc.numArcs - 1) * arc.bytesPerArc; + arc.nextArc = arc.posArcsStart() - (arc.numArcs() - 1) * arc.bytesPerArc(); } else { - arc.arcIdx = arc.numArcs - 2; + arc.arcIdx = arc.numArcs() - 2; } } else { arc.flags = b; @@ -844,25 +911,24 @@ public final class FST implements Accountable { if (follow.isFinal()) { // Insert "fake" final first arc: arc.label = END_LABEL; - arc.output = follow.nextFinalOutput; + arc.output = follow.nextFinalOutput(); arc.flags = BIT_FINAL_ARC; - if (follow.target <= 0) { + if (follow.target() <= 0) { arc.flags |= BIT_LAST_ARC; } else { // NOTE: nextArc is a node (not an address!) in this case: - arc.nextArc = follow.target; + arc.nextArc = follow.target(); } arc.target = FINAL_END_NODE; //System.out.println(" insert isFinal; nextArc=" + follow.target + " isLast=" + arc.isLast() + " output=" + outputs.outputToString(arc.output)); return arc; } else { - return readFirstRealTargetArc(follow.target, arc, in); + return readFirstRealTargetArc(follow.target(), arc, in); } } - public Arc readFirstRealTargetArc(long node, Arc arc, final BytesReader in) throws IOException { - final long address = node; - in.setPosition(address); + public Arc readFirstRealTargetArc(long nodeAddress, Arc arc, final BytesReader in) throws IOException { + in.setPosition(nodeAddress); //System.out.println(" flags=" + arc.flags); byte flags = in.readByte(); @@ -880,7 +946,7 @@ public final class FST implements Accountable { //System.out.println(" bytesPer=" + arc.bytesPerArc + " numArcs=" + arc.numArcs + " arcsStart=" + pos); } else { //arc.flags = b; - arc.nextArc = address; + arc.nextArc = nodeAddress; arc.bytesPerArc = 0; } @@ -897,7 +963,7 @@ public final class FST implements Accountable { if (!targetHasArcs(follow)) { return false; } else { - in.setPosition(follow.target); + in.setPosition(follow.target()); byte flags = in.readByte(); return flags == ARCS_AS_ARRAY_PACKED || flags == ARCS_AS_ARRAY_WITH_GAPS; } @@ -905,12 +971,12 @@ public final class FST implements Accountable { /** In-place read; returns the arc. */ public Arc readNextArc(Arc arc, BytesReader in) throws IOException { - if (arc.label == END_LABEL) { + if (arc.label() == END_LABEL) { // This was a fake inserted "final" arc - if (arc.nextArc <= 0) { + if (arc.nextArc() <= 0) { throw new IllegalArgumentException("cannot readNextArc when arc.isLast()=true"); } - return readFirstRealTargetArc(arc.nextArc, arc, in); + return readFirstRealTargetArc(arc.nextArc(), arc, in); } else { return readNextRealArc(arc, in); } @@ -918,14 +984,14 @@ public final class FST implements Accountable { /** Peeks at next arc's label; does not alter arc. Do * not call this if arc.isLast()! */ - public int readNextArcLabel(Arc arc, BytesReader in) throws IOException { + int readNextArcLabel(Arc arc, BytesReader in) throws IOException { assert !arc.isLast(); - if (arc.label == END_LABEL) { + if (arc.label() == END_LABEL) { //System.out.println(" nextArc fake " + //arc.nextArc); - long pos = arc.nextArc; + long pos = arc.nextArc(); in.setPosition(pos); final byte flags = in.readByte(); @@ -941,19 +1007,19 @@ public final class FST implements Accountable { // skip flags in.readByte(); } else { - if (arc.bytesPerArc != 0) { + if (arc.bytesPerArc() != 0) { //System.out.println(" nextArc real array"); // arcs are in an array - if (arc.arcIdx >= 0) { - in.setPosition(arc.posArcsStart); + if (arc.arcIdx() >= 0) { + in.setPosition(arc.posArcsStart()); // point at next arc, -1 to skip flags - in.skipBytes((1 + arc.arcIdx) * arc.bytesPerArc + 1); + in.skipBytes((1 + arc.arcIdx()) * arc.bytesPerArc() + 1); } else { - in.setPosition(arc.nextArc); + in.setPosition(arc.nextArc()); byte flags = in.readByte(); // skip missing arcs while (flag(flags, BIT_MISSING_ARC)) { - in.skipBytes(arc.bytesPerArc - 1); + in.skipBytes(arc.bytesPerArc() - 1); flags = in.readByte(); } } @@ -961,7 +1027,7 @@ public final class FST implements Accountable { // arcs are packed //System.out.println(" nextArc real packed"); // -1 to skip flags - in.setPosition(arc.nextArc - 1); + in.setPosition(arc.nextArc() - 1); } } return readLabel(in); @@ -975,29 +1041,30 @@ public final class FST implements Accountable { // assert !flag(arc.flags, BIT_LAST_ARC); // this is a continuing arc in a fixed array - if (arc.bytesPerArc != 0) { + if (arc.bytesPerArc() != 0) { // arcs are in an array - if (arc.arcIdx > Integer.MIN_VALUE) { + if (arc.arcIdx() > Integer.MIN_VALUE) { arc.arcIdx++; assert arc.arcIdx < arc.numArcs; - in.setPosition(arc.posArcsStart - arc.arcIdx * arc.bytesPerArc); + in.setPosition(arc.posArcsStart() - arc.arcIdx() * arc.bytesPerArc()); arc.flags = in.readByte(); } else { - assert arc.nextArc <= arc.posArcsStart && arc.nextArc > arc.posArcsStart - arc.numArcs * arc.bytesPerArc; - in.setPosition(arc.nextArc); + assert arc.nextArc() <= arc.posArcsStart() && arc.nextArc() > arc.posArcsStart() - arc.numArcs() * arc.bytesPerArc(); + in.setPosition(arc.nextArc()); arc.flags = in.readByte(); - while (flag(arc.flags, BIT_MISSING_ARC)) { + while (flag(arc.flags(), BIT_MISSING_ARC)) { // skip empty arcs - arc.nextArc -= arc.bytesPerArc; - in.skipBytes(arc.bytesPerArc - 1); + arc.nextArc = arc.nextArc() - arc.bytesPerArc(); + in.skipBytes(arc.bytesPerArc() - 1); arc.flags = in.readByte(); } } } else { // arcs are packed - in.setPosition(arc.nextArc); + in.setPosition(arc.nextArc()); arc.flags = in.readByte(); } + arc.label = readLabel(in); if (arc.flag(BIT_ARC_HAS_OUTPUT)) { @@ -1018,31 +1085,31 @@ public final class FST implements Accountable { } else { arc.target = NON_FINAL_END_NODE; } - if (arc.bytesPerArc == 0) { + if (arc.bytesPerArc() == 0) { arc.nextArc = in.getPosition(); } else { - arc.nextArc -= arc.bytesPerArc; + arc.nextArc -= arc.bytesPerArc(); } } else if (arc.flag(BIT_TARGET_NEXT)) { arc.nextArc = in.getPosition(); // TODO: would be nice to make this lazy -- maybe // caller doesn't need the target and is scanning arcs... if (!arc.flag(BIT_LAST_ARC)) { - if (arc.bytesPerArc == 0) { + if (arc.bytesPerArc() == 0) { // must scan seekToNextNode(in); } else { - in.setPosition(arc.posArcsStart); - in.skipBytes(arc.bytesPerArc * arc.numArcs); + in.setPosition(arc.posArcsStart()); + in.skipBytes(arc.bytesPerArc() * arc.numArcs()); } } arc.target = in.getPosition(); } else { arc.target = readUnpackedNodeTarget(in); - if (arc.bytesPerArc > 0 && arc.arcIdx == Integer.MIN_VALUE) { + if (arc.bytesPerArc() > 0 && arc.arcIdx() == Integer.MIN_VALUE) { // nextArc was pointing to *this* arc when we entered; advance to the next // if it is a missing arc, we will skip it later - arc.nextArc -= arc.bytesPerArc; + arc.nextArc = arc.nextArc() - arc.bytesPerArc(); } else { // in list and fixed table encodings, the next arc always follows this one arc.nextArc = in.getPosition(); @@ -1065,19 +1132,16 @@ public final class FST implements Accountable { assert cachedArc == null; } else { assert cachedArc != null; - assert cachedArc.arcIdx == result.arcIdx; - assert cachedArc.bytesPerArc == result.bytesPerArc; - assert cachedArc.flags == result.flags; - assert cachedArc.label == result.label; - if (cachedArc.bytesPerArc == 0 || cachedArc.arcIdx == Integer.MIN_VALUE) { - // in the sparse array case, this value is not valid, so don't assert it - assert cachedArc.nextArc == result.nextArc; - } - assert cachedArc.nextFinalOutput.equals(result.nextFinalOutput); - assert cachedArc.numArcs == result.numArcs; - assert cachedArc.output.equals(result.output); - assert cachedArc.posArcsStart == result.posArcsStart; - assert cachedArc.target == result.target; + assert cachedArc.arcIdx() == result.arcIdx(); + assert cachedArc.bytesPerArc() == result.bytesPerArc(); + assert cachedArc.flags() == result.flags(); + assert cachedArc.label() == result.label(); + assert (cachedArc.bytesPerArc() != 0 && cachedArc.arcIdx() != Integer.MIN_VALUE) || cachedArc.nextArc() == result.nextArc(); + assert cachedArc.nextFinalOutput().equals(result.nextFinalOutput()); + assert cachedArc.numArcs() == result.numArcs(); + assert cachedArc.output().equals(result.output()); + assert cachedArc.posArcsStart() == result.posArcsStart(); + assert cachedArc.target() == result.target(); } return true; @@ -1098,14 +1162,14 @@ public final class FST implements Accountable { if (labelToMatch == END_LABEL) { if (follow.isFinal()) { - if (follow.target <= 0) { + if (follow.target() <= 0) { arc.flags = BIT_LAST_ARC; } else { arc.flags = 0; // NOTE: nextArc is a node (not an address!) in this case: - arc.nextArc = follow.target; + arc.nextArc = follow.target(); } - arc.output = follow.nextFinalOutput; + arc.output = follow.nextFinalOutput(); arc.label = END_LABEL; return arc; } else { @@ -1114,7 +1178,7 @@ public final class FST implements Accountable { } // Short-circuit if this arc is in the root arc cache: - if (useRootArcCache && cachedRootArcs != null && follow.target == startNode && labelToMatch < cachedRootArcs.length) { + if (useRootArcCache && cachedRootArcs != null && follow.target() == startNode && labelToMatch < cachedRootArcs.length) { final Arc result = cachedRootArcs[labelToMatch]; // LUCENE-5152: detect tricky cases where caller @@ -1133,7 +1197,7 @@ public final class FST implements Accountable { return null; } - in.setPosition(follow.target); + in.setPosition(follow.target()); // System.out.println("fta label=" + (char) labelToMatch); @@ -1148,12 +1212,12 @@ public final class FST implements Accountable { int firstLabel = readLabel(in); int arcPos = labelToMatch - firstLabel; if (arcPos == 0) { - arc.nextArc = arc.posArcsStart; + arc.nextArc = arc.posArcsStart(); } else if (arcPos > 0) { - if (arcPos >= arc.numArcs) { + if (arcPos >= arc.numArcs()) { return null; } - in.setPosition(arc.posArcsStart - arc.bytesPerArc * arcPos); + in.setPosition(arc.posArcsStart() - arc.bytesPerArc() * arcPos); flags = in.readByte(); if (flag(flags, BIT_MISSING_ARC)) { return null; @@ -1172,12 +1236,12 @@ public final class FST implements Accountable { // Array is sparse; do binary search: int low = 0; - int high = arc.numArcs - 1; + int high = arc.numArcs() - 1; while (low <= high) { //System.out.println(" cycle"); int mid = (low + high) >>> 1; // +1 to skip over flags - in.setPosition(arc.posArcsStart - (arc.bytesPerArc * mid + 1)); + in.setPosition(arc.posArcsStart() - (arc.bytesPerArc() * mid + 1)); int midLabel = readLabel(in); final int cmp = midLabel - labelToMatch; if (cmp < 0) { @@ -1194,17 +1258,17 @@ public final class FST implements Accountable { } // Linear scan - readFirstRealTargetArc(follow.target, arc, in); + readFirstRealTargetArc(follow.target(), arc, in); while(true) { //System.out.println(" non-bs cycle"); // TODO: we should fix this code to not have to create // object for the output of every arc we scan... only // for the matching arc, if found - if (arc.label == labelToMatch) { + if (arc.label() == labelToMatch) { //System.out.println(" found!"); return arc; - } else if (arc.label > labelToMatch) { + } else if (arc.label() > labelToMatch) { return null; } else if (arc.isLast()) { return null; diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java index c7819a118bf..1c41d31f400 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java @@ -39,12 +39,12 @@ abstract class FSTEnum { protected final FST.Arc scratchArc = new FST.Arc<>(); protected int upto; - protected int targetLength; + int targetLength; /** doFloor controls the behavior of advance: if it's true * doFloor is true, advance positions to the biggest * term before target. */ - protected FSTEnum(FST fst) { + FSTEnum(FST fst) { this.fst = fst; fstReader = fst.getBytesReader(); NO_OUTPUT = fst.outputs.getNoOutput(); @@ -60,7 +60,7 @@ abstract class FSTEnum { /** Rewinds enum state to match the shared prefix between * current term and target term */ - protected final void rewindPrefix() throws IOException { + private void rewindPrefix() throws IOException { if (upto == 0) { //System.out.println(" init"); upto = 1; @@ -138,10 +138,10 @@ abstract class FSTEnum { while(arc != null) { int targetLabel = getTargetLabel(); //System.out.println(" cycle upto=" + upto + " arc.label=" + arc.label + " (" + (char) arc.label + ") vs targetLabel=" + targetLabel); - if (arc.bytesPerArc != 0 && arc.label != -1) { + if (arc.bytesPerArc() != 0 && arc.label() != -1) { // Arcs are in an array final FST.BytesReader in = fst.getBytesReader(); - if (arc.arcIdx == Integer.MIN_VALUE) { + if (arc.arcIdx() == Integer.MIN_VALUE) { arc = doSeekCeilArrayWithGaps(arc, targetLabel, in); } else { arc = doSeekCeilArrayPacked(arc, targetLabel, in); @@ -155,13 +155,13 @@ abstract class FSTEnum { private FST.Arc doSeekCeilArrayWithGaps(final FST.Arc arc, final int targetLabel, final FST.BytesReader in) throws IOException { // The array is addressed directly by label and may contain holes. - in.setPosition(arc.posArcsStart); + in.setPosition(arc.posArcsStart()); in.skipBytes(1); int firstLabel = fst.readLabel(in); int arcOffset = targetLabel - firstLabel; - if (arcOffset >= arc.numArcs) { + if (arcOffset >= arc.numArcs()) { // target is beyond the last arc - arc.nextArc = arc.posArcsStart - (arc.numArcs - 1) * arc.bytesPerArc; + arc.nextArc(arc.posArcsStart() - (arc.numArcs() - 1) * arc.bytesPerArc()); fst.readNextRealArc(arc, in); assert arc.isLast(); // Dead end (target is after the last arc); @@ -183,23 +183,23 @@ abstract class FSTEnum { } else { // TODO: if firstLabel == targetLabel if (arcOffset >= 0) { - arc.nextArc = arc.posArcsStart - (arc.bytesPerArc * arcOffset); + arc.nextArc(arc.posArcsStart() - (arc.bytesPerArc() * arcOffset)); } else { - arc.nextArc = arc.posArcsStart; + arc.nextArc(arc.posArcsStart()); } fst.readNextRealArc(arc, in); - if (arc.label == targetLabel) { + if (arc.label() == targetLabel) { // found -- copy pasta from below - output[upto] = fst.outputs.add(output[upto-1], arc.output); + output[upto] = fst.outputs.add(output[upto-1], arc.output()); if (targetLabel == FST.END_LABEL) { return null; } - setCurrentLabel(arc.label); + setCurrentLabel(arc.label()); incr(); return fst.readFirstTargetArc(arc, getArc(upto), fstReader); } // not found, return the next highest - assert arc.label > targetLabel; + assert arc.label() > targetLabel; pushFirst(); return null; } @@ -208,15 +208,15 @@ abstract class FSTEnum { private FST.Arc doSeekCeilArrayPacked(final FST.Arc arc, final int targetLabel, final FST.BytesReader in) throws IOException { // The array is packed -- use binary search to find the target. - int low = arc.arcIdx; - int high = arc.numArcs-1; + int low = arc.arcIdx(); + int high = arc.numArcs() -1; int mid = 0; //System.out.println("do arc array low=" + low + " high=" + high + " targetLabel=" + targetLabel); boolean found = false; while (low <= high) { mid = (low + high) >>> 1; - in.setPosition(arc.posArcsStart); - in.skipBytes(arc.bytesPerArc * mid + 1); + in.setPosition(arc.posArcsStart()); + in.skipBytes(arc.bytesPerArc() * mid + 1); final int midLabel = fst.readLabel(in); final int cmp = midLabel - targetLabel; //System.out.println(" cycle low=" + low + " high=" + high + " mid=" + mid + " midLabel=" + midLabel + " cmp=" + cmp); @@ -234,20 +234,20 @@ abstract class FSTEnum { // the outer else clause): if (found) { // Match - arc.arcIdx = mid-1; + arc.arcIdx(mid - 1); fst.readNextRealArc(arc, in); - assert arc.arcIdx == mid; - assert arc.label == targetLabel: "arc.label=" + arc.label + " vs targetLabel=" + targetLabel + " mid=" + mid; - output[upto] = fst.outputs.add(output[upto-1], arc.output); + assert arc.arcIdx() == mid; + assert arc.label() == targetLabel: "arc.label=" + arc.label() + " vs targetLabel=" + targetLabel + " mid=" + mid; + output[upto] = fst.outputs.add(output[upto-1], arc.output()); if (targetLabel == FST.END_LABEL) { return null; } - setCurrentLabel(arc.label); + setCurrentLabel(arc.label()); incr(); return fst.readFirstTargetArc(arc, getArc(upto), fstReader); - } else if (low == arc.numArcs) { + } else if (low == arc.numArcs()) { // Dead end - arc.arcIdx = arc.numArcs-2; + arc.arcIdx(arc.numArcs() - 2); fst.readNextRealArc(arc, in); assert arc.isLast(); // Dead end (target is after the last arc); @@ -267,9 +267,9 @@ abstract class FSTEnum { upto--; } } else { - arc.arcIdx = (low > high ? low : high)-1; + arc.arcIdx(low - 1); fst.readNextRealArc(arc, in); - assert arc.label > targetLabel; + assert arc.label() > targetLabel; pushFirst(); return null; } @@ -277,16 +277,16 @@ abstract class FSTEnum { private FST.Arc doSeekCeilList(final FST.Arc arc, final int targetLabel) throws IOException { // Arcs are not array'd -- must do linear scan: - if (arc.label == targetLabel) { + if (arc.label() == targetLabel) { // recurse - output[upto] = fst.outputs.add(output[upto-1], arc.output); + output[upto] = fst.outputs.add(output[upto-1], arc.output()); if (targetLabel == FST.END_LABEL) { return null; } - setCurrentLabel(arc.label); + setCurrentLabel(arc.label()); incr(); return fst.readFirstTargetArc(arc, getArc(upto), fstReader); - } else if (arc.label > targetLabel) { + } else if (arc.label() > targetLabel) { pushFirst(); return null; } else if (arc.isLast()) { @@ -340,10 +340,10 @@ abstract class FSTEnum { //System.out.println(" cycle upto=" + upto + " arc.label=" + arc.label + " (" + (char) arc.label + ") targetLabel=" + targetLabel + " isLast?=" + arc.isLast() + " bba=" + arc.bytesPerArc); int targetLabel = getTargetLabel(); - if (arc.bytesPerArc != 0 && arc.label != FST.END_LABEL) { + if (arc.bytesPerArc() != 0 && arc.label() != FST.END_LABEL) { // Arcs are in an array final FST.BytesReader in = fst.getBytesReader(); - if (arc.arcIdx == Integer.MIN_VALUE) { + if (arc.arcIdx() == Integer.MIN_VALUE) { arc = doSeekFloorArrayWithGaps(arc, targetLabel, in); } else { arc = doSeekFloorArrayPacked(arc, targetLabel, in); @@ -356,7 +356,7 @@ abstract class FSTEnum { private FST.Arc doSeekFloorArrayWithGaps(FST.Arc arc, int targetLabel, final FST.BytesReader in) throws IOException { // The array is addressed directly by label and may contain holes. - in.setPosition(arc.posArcsStart); + in.setPosition(arc.posArcsStart()); in.skipBytes(1); int firstLabel = fst.readLabel(in); int targetOffset = targetLabel - firstLabel; @@ -368,7 +368,7 @@ abstract class FSTEnum { // First, walk backwards until we find a first arc // that's before our target label: fst.readFirstTargetArc(getArc(upto-1), arc, fstReader); - if (arc.label < targetLabel) { + if (arc.label() < targetLabel) { // Then, scan forwards to the arc just before // the targetLabel: while(!arc.isLast() && fst.readNextArcLabel(arc, in) < targetLabel) { @@ -385,39 +385,38 @@ abstract class FSTEnum { arc = getArc(upto); } } else { - if (targetOffset >= arc.numArcs) { - arc.nextArc = arc.posArcsStart - arc.bytesPerArc * (arc.numArcs - 1); + if (targetOffset >= arc.numArcs()) { + arc.nextArc(arc.posArcsStart() - arc.bytesPerArc() * (arc.numArcs() - 1)); fst.readNextRealArc(arc, in); assert arc.isLast(); - assert arc.label < targetLabel: "arc.label=" + arc.label + " vs targetLabel=" + targetLabel; + assert arc.label() < targetLabel: "arc.label=" + arc.label() + " vs targetLabel=" + targetLabel; pushLast(); return null; } - arc.nextArc = arc.posArcsStart - arc.bytesPerArc * targetOffset; + arc.nextArc(arc.posArcsStart() - arc.bytesPerArc() * targetOffset); fst.readNextRealArc(arc, in); - if (arc.label == targetLabel) { + if (arc.label() == targetLabel) { // found -- copy pasta from below - output[upto] = fst.outputs.add(output[upto-1], arc.output); + output[upto] = fst.outputs.add(output[upto-1], arc.output()); if (targetLabel == FST.END_LABEL) { return null; } - setCurrentLabel(arc.label); + setCurrentLabel(arc.label()); incr(); return fst.readFirstTargetArc(arc, getArc(upto), fstReader); } // Scan backwards to find a floor arc that is not missing - for (long arcOffset = arc.posArcsStart - targetOffset * arc.bytesPerArc; arcOffset <= arc.posArcsStart; arcOffset += arc.bytesPerArc) { + for (long arcOffset = arc.posArcsStart() - targetOffset * arc.bytesPerArc(); arcOffset <= arc.posArcsStart(); arcOffset += arc.bytesPerArc()) { // TODO: we can do better here by skipping missing arcs - arc.nextArc = arcOffset; - //System.out.println(" hasFloor arcIdx=" + (arc.arcIdx+1)); + arc.nextArc(arcOffset); fst.readNextRealArc(arc, in); - if (arc.label < targetLabel) { + if (arc.label() < targetLabel) { assert arc.isLast() || fst.readNextArcLabel(arc, in) > targetLabel; pushLast(); return null; } } - assert false: "arc.label=" + arc.label + " vs targetLabel=" + targetLabel; + assert false: "arc.label=" + arc.label() + " vs targetLabel=" + targetLabel; return arc; // unreachable } } @@ -425,15 +424,15 @@ abstract class FSTEnum { private FST.Arc doSeekFloorArrayPacked(FST.Arc arc, int targetLabel, final FST.BytesReader in) throws IOException { // Arcs are fixed array -- use binary search to find the target. - int low = arc.arcIdx; - int high = arc.numArcs-1; + int low = arc.arcIdx(); + int high = arc.numArcs() -1; int mid = 0; //System.out.println("do arc array low=" + low + " high=" + high + " targetLabel=" + targetLabel); boolean found = false; while (low <= high) { mid = (low + high) >>> 1; - in.setPosition(arc.posArcsStart); - in.skipBytes(arc.bytesPerArc*mid+1); + in.setPosition(arc.posArcsStart()); + in.skipBytes(arc.bytesPerArc() * mid + 1); final int midLabel = fst.readLabel(in); final int cmp = midLabel - targetLabel; //System.out.println(" cycle low=" + low + " high=" + high + " mid=" + mid + " midLabel=" + midLabel + " cmp=" + cmp); @@ -452,15 +451,15 @@ abstract class FSTEnum { if (found) { // Match -- recurse //System.out.println(" match! arcIdx=" + mid); - arc.arcIdx = mid-1; + arc.arcIdx(mid - 1); fst.readNextRealArc(arc, in); - assert arc.arcIdx == mid; - assert arc.label == targetLabel: "arc.label=" + arc.label + " vs targetLabel=" + targetLabel + " mid=" + mid; - output[upto] = fst.outputs.add(output[upto-1], arc.output); + assert arc.arcIdx() == mid; + assert arc.label() == targetLabel: "arc.label=" + arc.label() + " vs targetLabel=" + targetLabel + " mid=" + mid; + output[upto] = fst.outputs.add(output[upto-1], arc.output()); if (targetLabel == FST.END_LABEL) { return null; } - setCurrentLabel(arc.label); + setCurrentLabel(arc.label()); incr(); return fst.readFirstTargetArc(arc, getArc(upto), fstReader); } else if (high == -1) { @@ -474,7 +473,7 @@ abstract class FSTEnum { // First, walk backwards until we find a first arc // that's before our target label: fst.readFirstTargetArc(getArc(upto-1), arc, fstReader); - if (arc.label < targetLabel) { + if (arc.label() < targetLabel) { // Then, scan forwards to the arc just before // the targetLabel: while(!arc.isLast() && fst.readNextArcLabel(arc, in) < targetLabel) { @@ -492,27 +491,26 @@ abstract class FSTEnum { } } else { // There is a floor arc: - arc.arcIdx = (low > high ? high : low)-1; - //System.out.println(" hasFloor arcIdx=" + (arc.arcIdx+1)); + arc.arcIdx(high - 1); fst.readNextRealArc(arc, in); assert arc.isLast() || fst.readNextArcLabel(arc, in) > targetLabel; - assert arc.label < targetLabel: "arc.label=" + arc.label + " vs targetLabel=" + targetLabel; + assert arc.label() < targetLabel: "arc.label=" + arc.label() + " vs targetLabel=" + targetLabel; pushLast(); return null; } } private FST.Arc doSeekFloorList(FST.Arc arc, int targetLabel) throws IOException { - if (arc.label == targetLabel) { + if (arc.label() == targetLabel) { // Match -- recurse - output[upto] = fst.outputs.add(output[upto-1], arc.output); + output[upto] = fst.outputs.add(output[upto-1], arc.output()); if (targetLabel == FST.END_LABEL) { return null; } - setCurrentLabel(arc.label); + setCurrentLabel(arc.label()); incr(); return fst.readFirstTargetArc(arc, getArc(upto), fstReader); - } else if (arc.label > targetLabel) { + } else if (arc.label() > targetLabel) { // TODO: if each arc could somehow read the arc just // before, we can save this re-scan. The ceil case // doesn't need this because it reads the next arc @@ -521,7 +519,7 @@ abstract class FSTEnum { // First, walk backwards until we find a first arc // that's before our target label: fst.readFirstTargetArc(getArc(upto-1), arc, fstReader); - if (arc.label < targetLabel) { + if (arc.label() < targetLabel) { // Then, scan forwards to the arc just before // the targetLabel: while(!arc.isLast() && fst.readNextArcLabel(arc, fstReader) < targetLabel) { @@ -553,7 +551,7 @@ abstract class FSTEnum { } /** Seeks to exactly target term. */ - protected boolean doSeekExact() throws IOException { + boolean doSeekExact() throws IOException { // TODO: possibly caller could/should provide common // prefix length? ie this work may be redundant if @@ -584,7 +582,7 @@ abstract class FSTEnum { return false; } // Match -- recurse: - output[upto] = fst.outputs.add(output[upto-1], nextArc.output); + output[upto] = fst.outputs.add(output[upto-1], nextArc.output()); if (targetLabel == FST.END_LABEL) { //System.out.println(" return found; upto=" + upto + " output=" + output[upto] + " nextArc=" + nextArc.isLast()); return true; @@ -621,13 +619,13 @@ abstract class FSTEnum { assert arc != null; while (true) { - output[upto] = fst.outputs.add(output[upto-1], arc.output); - if (arc.label == FST.END_LABEL) { + output[upto] = fst.outputs.add(output[upto-1], arc.output()); + if (arc.label() == FST.END_LABEL) { // Final node break; } //System.out.println(" pushFirst label=" + (char) arc.label + " upto=" + upto + " output=" + fst.outputs.outputToString(output[upto])); - setCurrentLabel(arc.label); + setCurrentLabel(arc.label()); incr(); final FST.Arc nextArc = getArc(upto); @@ -644,9 +642,9 @@ abstract class FSTEnum { assert arc != null; while (true) { - setCurrentLabel(arc.label); - output[upto] = fst.outputs.add(output[upto-1], arc.output); - if (arc.label == FST.END_LABEL) { + setCurrentLabel(arc.label()); + output[upto] = fst.outputs.add(output[upto-1], arc.output()); + if (arc.label() == FST.END_LABEL) { // Final node break; } diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java b/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java index c7023c13751..0de8c97b136 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java @@ -41,15 +41,15 @@ final class NodeHash { private boolean nodesEqual(Builder.UnCompiledNode node, long address) throws IOException { fst.readFirstRealTargetArc(address, scratchArc, in); - if (scratchArc.bytesPerArc != 0 && node.numArcs != scratchArc.numArcs) { + if (scratchArc.bytesPerArc() != 0 && node.numArcs != scratchArc.numArcs()) { return false; } for(int arcUpto=0;arcUpto arc = node.arcs[arcUpto]; - if (arc.label != scratchArc.label || - !arc.output.equals(scratchArc.output) || - ((Builder.CompiledNode) arc.target).node != scratchArc.target || - !arc.nextFinalOutput.equals(scratchArc.nextFinalOutput) || + if (arc.label != scratchArc.label() || + !arc.output.equals(scratchArc.output()) || + ((Builder.CompiledNode) arc.target).node != scratchArc.target() || + !arc.nextFinalOutput.equals(scratchArc.nextFinalOutput()) || arc.isFinal != scratchArc.isFinal()) { return false; } @@ -98,10 +98,10 @@ final class NodeHash { fst.readFirstRealTargetArc(node, scratchArc, in); while(true) { // System.out.println(" label=" + scratchArc.label + " target=" + scratchArc.target + " h=" + h + " output=" + fst.outputs.outputToString(scratchArc.output) + " next?=" + scratchArc.flag(4) + " final?=" + scratchArc.isFinal() + " pos=" + in.getPosition()); - h = PRIME * h + scratchArc.label; - h = PRIME * h + (int) (scratchArc.target^(scratchArc.target>>32)); - h = PRIME * h + scratchArc.output.hashCode(); - h = PRIME * h + scratchArc.nextFinalOutput.hashCode(); + h = PRIME * h + scratchArc.label(); + h = PRIME * h + (int) (scratchArc.target() ^(scratchArc.target() >>32)); + h = PRIME * h + scratchArc.output().hashCode(); + h = PRIME * h + scratchArc.nextFinalOutput().hashCode(); if (scratchArc.isFinal()) { h += 17; } diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/Util.java b/lucene/core/src/java/org/apache/lucene/util/fst/Util.java index d435bcfdd77..0b2a0eea503 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/Util.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/Util.java @@ -55,11 +55,11 @@ public final class Util { if (fst.findTargetArc(input.ints[input.offset + i], arc, arc, fstReader) == null) { return null; } - output = fst.outputs.add(output, arc.output); + output = fst.outputs.add(output, arc.output()); } if (arc.isFinal()) { - return fst.outputs.add(output, arc.nextFinalOutput); + return fst.outputs.add(output, arc.nextFinalOutput()); } else { return null; } @@ -75,7 +75,7 @@ public final class Util { final BytesReader fstReader = fst.getBytesReader(); // TODO: would be nice not to alloc this on every lookup - final FST.Arc arc = fst.getFirstArc(new FST.Arc()); + final FST.Arc arc = fst.getFirstArc(new FST.Arc<>()); // Accumulate output as we go T output = fst.outputs.getNoOutput(); @@ -83,11 +83,11 @@ public final class Util { if (fst.findTargetArc(input.bytes[i+input.offset] & 0xFF, arc, arc, fstReader) == null) { return null; } - output = fst.outputs.add(output, arc.output); + output = fst.outputs.add(output, arc.output()); } if (arc.isFinal()) { - return fst.outputs.add(output, arc.nextFinalOutput); + return fst.outputs.add(output, arc.nextFinalOutput()); } else { return null; } @@ -125,7 +125,7 @@ public final class Util { */ @Deprecated public static IntsRef getByOutput(FST fst, long targetOutput, BytesReader in, Arc arc, Arc scratchArc, IntsRefBuilder result) throws IOException { - long output = arc.output; + long output = arc.output(); int upto = 0; //System.out.println("reverseLookup output=" + targetOutput); @@ -133,7 +133,7 @@ public final class Util { while(true) { //System.out.println("loop: output=" + output + " upto=" + upto + " arc=" + arc); if (arc.isFinal()) { - final long finalOutput = output + arc.nextFinalOutput; + final long finalOutput = output + arc.nextFinalOutput(); //System.out.println(" isFinal finalOutput=" + finalOutput); if (finalOutput == targetOutput) { result.setLength(upto); @@ -149,19 +149,19 @@ public final class Util { //System.out.println(" targetHasArcs"); result.grow(1+upto); - fst.readFirstRealTargetArc(arc.target, arc, in); + fst.readFirstRealTargetArc(arc.target(), arc, in); - if (arc.bytesPerArc != 0 && arc.arcIdx > Integer.MIN_VALUE) { + if (arc.bytesPerArc() != 0 && arc.arcIdx() > Integer.MIN_VALUE) { int low = 0; - int high = arc.numArcs-1; + int high = arc.numArcs() -1; int mid = 0; //System.out.println("bsearch: numArcs=" + arc.numArcs + " target=" + targetOutput + " output=" + output); boolean exact = false; while (low <= high) { mid = (low + high) >>> 1; - in.setPosition(arc.posArcsStart); - in.skipBytes(arc.bytesPerArc*mid); + in.setPosition(arc.posArcsStart()); + in.skipBytes(arc.bytesPerArc() *mid); final byte flags = in.readByte(); fst.readLabel(in); final long minArcOutput; @@ -182,17 +182,19 @@ public final class Util { } } + int idx; if (high == -1) { return null; } else if (exact) { - arc.arcIdx = mid-1; + idx = mid; } else { - arc.arcIdx = low-2; + idx = low - 1; } + arc.arcIdx(idx - 1); fst.readNextRealArc(arc, in); - result.setIntAt(upto++, arc.label); - output += arc.output; + result.setIntAt(upto++, arc.label()); + output += arc.output(); } else { @@ -203,13 +205,13 @@ public final class Util { // This is the min output we'd hit if we follow // this arc: - final long minArcOutput = output + arc.output; + final long minArcOutput = output + arc.output(); if (minArcOutput == targetOutput) { // Recurse on this arc: //System.out.println(" match! break"); output = minArcOutput; - result.setIntAt(upto++, arc.label); + result.setIntAt(upto++, arc.label()); break; } else if (minArcOutput > targetOutput) { if (prevArc == null) { @@ -218,8 +220,8 @@ public final class Util { } else { // Recurse on previous arc: arc.copyFrom(prevArc); - result.setIntAt(upto++, arc.label); - output += arc.output; + result.setIntAt(upto++, arc.label()); + output += arc.output(); //System.out.println(" recurse prev label=" + (char) arc.label + " output=" + output); break; } @@ -227,7 +229,7 @@ public final class Util { // Recurse on this arc: output = minArcOutput; //System.out.println(" recurse last label=" + (char) arc.label + " output=" + output); - result.setIntAt(upto++, arc.label); + result.setIntAt(upto++, arc.label()); break; } else { // Read next arc in this node: @@ -261,12 +263,7 @@ public final class Util { // Custom int payload for consumers; the NRT suggester uses this to record if this path has already enumerated a surface form public int payload; - /** Sole constructor */ - public FSTPath(T output, FST.Arc arc, IntsRefBuilder input) { - this(output, arc, input, 0, null, -1); - } - - public FSTPath(T output, FST.Arc arc, IntsRefBuilder input, float boost, CharSequence context, int payload) { + FSTPath(T output, FST.Arc arc, IntsRefBuilder input, float boost, CharSequence context, int payload) { this.arc = new FST.Arc().copyFrom(arc); this.output = output; this.input = input; @@ -275,7 +272,7 @@ public final class Util { this.payload = payload; } - public FSTPath newPath(T output, IntsRefBuilder input) { + FSTPath newPath(T output, IntsRefBuilder input) { return new FSTPath<>(output, this.arc, input, this.boost, this.context, this.payload); } @@ -289,7 +286,8 @@ public final class Util { * tie breaks by path.input. */ private static class TieBreakByInputComparator implements Comparator> { private final Comparator comparator; - public TieBreakByInputComparator(Comparator comparator) { + + TieBreakByInputComparator(Comparator comparator) { this.comparator = comparator; } @@ -318,7 +316,7 @@ public final class Util { private final Comparator comparator; private final Comparator> pathComparator; - TreeSet> queue = null; + TreeSet> queue; /** * Creates an unbounded TopNSearcher @@ -347,7 +345,7 @@ public final class Util { assert queue != null; - T output = fst.outputs.add(path.output, path.arc.output); + T output = fst.outputs.add(path.output, path.arc.output()); if (queue.size() == maxQueueDepth) { FSTPath bottom = queue.last(); @@ -357,7 +355,7 @@ public final class Util { return; } else if (comp == 0) { // Tie break by alpha sort on the input: - path.input.append(path.arc.label); + path.input.append(path.arc.label()); final int cmp = bottom.input.get().compareTo(path.input.get()); path.input.setLength(path.input.length() - 1); @@ -370,15 +368,14 @@ public final class Util { } } // Competes - } else { - // Queue isn't full yet, so any path we hit competes: } + // else ... Queue isn't full yet, so any path we hit competes: // copy over the current input to the new input // and add the arc.label to the end IntsRefBuilder newInput = new IntsRefBuilder(); newInput.copyInts(path.input.get()); - newInput.append(path.arc.label); + newInput.append(path.arc.label()); FSTPath newPath = path.newPath(output, newInput); if (acceptPartialPath(newPath)) { @@ -408,7 +405,7 @@ public final class Util { // Bootstrap: find the min starting arc while (true) { - if (allowEmptyString || path.arc.label != FST.END_LABEL) { + if (allowEmptyString || path.arc.label() != FST.END_LABEL) { addIfCompetitive(path); } if (path.arc.isLast()) { @@ -457,7 +454,7 @@ public final class Util { continue; } - if (path.arc.label == FST.END_LABEL) { + if (path.arc.label() == FST.END_LABEL) { // Empty string! path.input.setLength(path.input.length() - 1); results.add(new Result<>(path.input.get(), path.output)); @@ -485,7 +482,7 @@ public final class Util { while(true) { // tricky: instead of comparing output == 0, we must // express it via the comparator compare(output, 0) == 0 - if (comparator.compare(NO_OUTPUT, path.arc.output) == 0) { + if (comparator.compare(NO_OUTPUT, path.arc.output()) == 0) { if (queue == null) { foundZero = true; break; @@ -514,9 +511,9 @@ public final class Util { path.arc.copyFrom(scratchArc); } - if (path.arc.label == FST.END_LABEL) { + if (path.arc.label() == FST.END_LABEL) { // Add final output: - path.output = fst.outputs.add(path.output, path.arc.output); + path.output = fst.outputs.add(path.output, path.arc.output()); if (acceptResult(path)) { results.add(new Result<>(path.input.get(), path.output)); } else { @@ -524,8 +521,8 @@ public final class Util { } break; } else { - path.input.append(path.arc.label); - path.output = fst.outputs.add(path.output, path.arc.output); + path.input.append(path.arc.label()); + path.output = fst.outputs.add(path.output, path.arc.output()); if (acceptPartialPath(path) == false) { break; } @@ -641,7 +638,7 @@ public final class Util { // This is the start arc in the automaton (from the epsilon state to the first state // with outgoing transitions. - final FST.Arc startArc = fst.getFirstArc(new FST.Arc()); + final FST.Arc startArc = fst.getFirstArc(new FST.Arc<>()); // A queue of transitions to consider for the next level. final List> thisLevelQueue = new ArrayList<>(); @@ -656,7 +653,7 @@ public final class Util { // A bitset of already seen states (target offset). final BitSet seen = new BitSet(); - seen.set((int) startArc.target); + seen.set((int) startArc.target()); // Shape for states. final String stateShape = "circle"; @@ -689,16 +686,16 @@ public final class Util { final T finalOutput; if (startArc.isFinal()) { isFinal = true; - finalOutput = startArc.nextFinalOutput == NO_OUTPUT ? null : startArc.nextFinalOutput; + finalOutput = startArc.nextFinalOutput() == NO_OUTPUT ? null : startArc.nextFinalOutput(); } else { isFinal = false; finalOutput = null; } - emitDotState(out, Long.toString(startArc.target), isFinal ? finalStateShape : stateShape, stateColor, finalOutput == null ? "" : fst.outputs.outputToString(finalOutput)); + emitDotState(out, Long.toString(startArc.target()), isFinal ? finalStateShape : stateShape, stateColor, finalOutput == null ? "" : fst.outputs.outputToString(finalOutput)); } - out.write(" initial -> " + startArc.target + "\n"); + out.write(" initial -> " + startArc.target() + "\n"); int level = 0; @@ -717,9 +714,9 @@ public final class Util { // scan all target arcs //System.out.println(" readFirstTarget..."); - final long node = arc.target; + final long node = arc.target(); - fst.readFirstRealTargetArc(arc.target, arc, r); + fst.readFirstRealTargetArc(arc.target(), arc, r); //System.out.println(" firstTarget: " + arc); @@ -727,7 +724,7 @@ public final class Util { //System.out.println(" cycle arc=" + arc); // Emit the unseen state and add it to the queue for the next level. - if (arc.target >= 0 && !seen.get((int) arc.target)) { + if (arc.target() >= 0 && !seen.get((int) arc.target())) { /* boolean isFinal = false; @@ -748,35 +745,35 @@ public final class Util { } final String finalOutput; - if (arc.nextFinalOutput != null && arc.nextFinalOutput != NO_OUTPUT) { - finalOutput = fst.outputs.outputToString(arc.nextFinalOutput); + if (arc.nextFinalOutput() != null && arc.nextFinalOutput() != NO_OUTPUT) { + finalOutput = fst.outputs.outputToString(arc.nextFinalOutput()); } else { finalOutput = ""; } - emitDotState(out, Long.toString(arc.target), stateShape, stateColor, finalOutput); + emitDotState(out, Long.toString(arc.target()), stateShape, stateColor, finalOutput); // To see the node address, use this instead: //emitDotState(out, Integer.toString(arc.target), stateShape, stateColor, String.valueOf(arc.target)); - seen.set((int) arc.target); + seen.set((int) arc.target()); nextLevelQueue.add(new FST.Arc().copyFrom(arc)); - sameLevelStates.add((int) arc.target); + sameLevelStates.add((int) arc.target()); } String outs; - if (arc.output != NO_OUTPUT) { - outs = "/" + fst.outputs.outputToString(arc.output); + if (arc.output() != NO_OUTPUT) { + outs = "/" + fst.outputs.outputToString(arc.output()); } else { outs = ""; } - if (!FST.targetHasArcs(arc) && arc.isFinal() && arc.nextFinalOutput != NO_OUTPUT) { + if (!FST.targetHasArcs(arc) && arc.isFinal() && arc.nextFinalOutput() != NO_OUTPUT) { // Tricky special case: sometimes, due to // pruning, the builder can [sillily] produce // an FST with an arc into the final end state // (-1) but also with a next final output; in // this case we pull that output up onto this // arc - outs = outs + "/[" + fst.outputs.outputToString(arc.nextFinalOutput) + "]"; + outs = outs + "/[" + fst.outputs.outputToString(arc.nextFinalOutput()) + "]"; } final String arcColor; @@ -786,8 +783,8 @@ public final class Util { arcColor = "black"; } - assert arc.label != FST.END_LABEL; - out.write(" " + node + " -> " + arc.target + " [label=\"" + printableLabel(arc.label) + outs + "\"" + (arc.isFinal() ? " style=\"bold\"" : "" ) + " color=\"" + arcColor + "\"]\n"); + assert arc.label() != FST.END_LABEL; + out.write(" " + node + " -> " + arc.target() + " [label=\"" + printableLabel(arc.label()) + outs + "\"" + (arc.isFinal() ? " style=\"bold\"" : "" ) + " color=\"" + arcColor + "\"]\n"); // Break the loop if we're on the last arc of this state. if (arc.isLast()) { @@ -935,55 +932,52 @@ public final class Util { * @param arc the arc to read into in place * @param in the fst's {@link BytesReader} */ - public static Arc readCeilArc(int label, FST fst, Arc follow, - Arc arc, BytesReader in) throws IOException { - // TODO maybe this is a useful in the FST class - we could simplify some other code like FSTEnum? + public static Arc readCeilArc(int label, FST fst, Arc follow, Arc arc, BytesReader in) throws IOException { if (label == FST.END_LABEL) { if (follow.isFinal()) { - if (follow.target <= 0) { - arc.flags = FST.BIT_LAST_ARC; + if (follow.target() <= 0) { + arc.flags((byte) FST.BIT_LAST_ARC); } else { - arc.flags = 0; + arc.flags((byte) 0); // NOTE: nextArc is a node (not an address!) in this case: - arc.nextArc = follow.target; + arc.nextArc(follow.target()); } - arc.output = follow.nextFinalOutput; - arc.label = FST.END_LABEL; + arc.output(follow.nextFinalOutput()); + arc.label(FST.END_LABEL); return arc; } else { return null; } } - if (!FST.targetHasArcs(follow)) { return null; } fst.readFirstTargetArc(follow, arc, in); - if (arc.bytesPerArc != 0 && arc.label != FST.END_LABEL) { - if (arc.arcIdx == Integer.MIN_VALUE) { + if (arc.bytesPerArc() != 0 && arc.label() != FST.END_LABEL) { + if (arc.arcIdx() == Integer.MIN_VALUE) { // Arcs are in an array-with-gaps - int offset = label - arc.label; - if (offset >= arc.numArcs) { + int offset = label - arc.label(); + if (offset >= arc.numArcs()) { return null; } else if (offset < 0) { return arc; } else { - arc.nextArc = arc.posArcsStart - offset * arc.bytesPerArc; + arc.nextArc(arc.posArcsStart() - offset * arc.bytesPerArc()); return fst.readNextRealArc(arc, in); } } // Arcs are packed array -- use binary search to find // the target. - int low = arc.arcIdx; - int high = arc.numArcs - 1; + int low = arc.arcIdx(); int mid = 0; + int high = arc.numArcs() - 1; // System.out.println("do arc array low=" + low + " high=" + high + // " targetLabel=" + targetLabel); while (low <= high) { mid = (low + high) >>> 1; - in.setPosition(arc.posArcsStart); - in.skipBytes(arc.bytesPerArc * mid + 1); + in.setPosition(arc.posArcsStart()); + in.skipBytes(arc.bytesPerArc() * mid + 1); final int midLabel = fst.readLabel(in); final int cmp = midLabel - label; // System.out.println(" cycle low=" + low + " high=" + high + " mid=" + @@ -993,28 +987,27 @@ public final class Util { } else if (cmp > 0) { high = mid - 1; } else { - arc.arcIdx = mid-1; + arc.arcIdx(mid - 1); return fst.readNextRealArc(arc, in); } } - if (low == arc.numArcs) { + if (low == arc.numArcs()) { // DEAD END! return null; } - - arc.arcIdx = (low > high ? high : low); - return fst.readNextRealArc(arc, in); + arc.arcIdx(high + 1); + return fst.readNextRealArc(arc, in ); } // Linear scan - fst.readFirstRealTargetArc(follow.target, arc, in); + fst.readFirstRealTargetArc(follow.target(), arc, in); while (true) { // System.out.println(" non-bs cycle"); // TODO: we should fix this code to not have to create // object for the output of every arc we scan... only // for the matching arc, if found - if (arc.label >= label) { + if (arc.label() >= label) { // System.out.println(" found!"); return arc; } else if (arc.isLast()) { @@ -1024,4 +1017,5 @@ public final class Util { } } } + } diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java index d882fae394d..8f598f63fb0 100644 --- a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java +++ b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java @@ -1201,19 +1201,19 @@ public class TestFSTs extends LuceneTestCase { private void checkStopNodes(FST fst, PositiveIntOutputs outputs) throws Exception { final Long nothing = outputs.getNoOutput(); FST.Arc startArc = fst.getFirstArc(new FST.Arc()); - assertEquals(nothing, startArc.output); - assertEquals(nothing, startArc.nextFinalOutput); + assertEquals(nothing, startArc.output()); + assertEquals(nothing, startArc.nextFinalOutput()); FST.Arc arc = fst.readFirstTargetArc(startArc, new FST.Arc(), fst.getBytesReader()); - assertEquals('a', arc.label); - assertEquals(17, arc.nextFinalOutput.longValue()); + assertEquals('a', arc.label()); + assertEquals(17, arc.nextFinalOutput().longValue()); assertTrue(arc.isFinal()); arc = fst.readNextArc(arc, fst.getBytesReader()); - assertEquals('b', arc.label); + assertEquals('b', arc.label()); assertFalse(arc.isFinal()); - assertEquals(42, arc.output.longValue()); + assertEquals(42, arc.output().longValue()); } static final Comparator minLongComparator = new Comparator () { @@ -1404,7 +1404,7 @@ public class TestFSTs extends LuceneTestCase { if (fst.findTargetArc((int) prefix.charAt(idx), arc, arc, reader) == null) { fail(); } - prefixOutput += arc.output; + prefixOutput += arc.output(); } final int topN = TestUtil.nextInt(random, 1, 10); @@ -1526,7 +1526,7 @@ public class TestFSTs extends LuceneTestCase { if (fst.findTargetArc((int) prefix.charAt(idx), arc, arc, reader) == null) { fail(); } - prefixOutput = outputs.add(prefixOutput, arc.output); + prefixOutput = outputs.add(prefixOutput, arc.output()); } final int topN = TestUtil.nextInt(random, 1, 10); @@ -1623,10 +1623,10 @@ public class TestFSTs extends LuceneTestCase { FST.BytesReader reader = fst.getBytesReader(); arc = fst.findTargetArc((int) 'm', arc, arc, reader); assertNotNull(arc); - assertEquals(new BytesRef("m"), arc.output); + assertEquals(new BytesRef("m"), arc.output()); // NOTE: illegal: - arc.output.length = 0; + arc.output().length = 0; fst.getFirstArc(arc); try { diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/TestFstDirect.java b/lucene/core/src/test/org/apache/lucene/util/fst/TestFstDirect.java index 5ec44416768..8763d0416d7 100644 --- a/lucene/core/src/test/org/apache/lucene/util/fst/TestFstDirect.java +++ b/lucene/core/src/test/org/apache/lucene/util/fst/TestFstDirect.java @@ -79,9 +79,9 @@ public class TestFstDirect extends LuceneTestCase { BytesRefFSTEnum fstEnum = new BytesRefFSTEnum<>(fst); int sparseArrayArcCount = 0, directArrayArcCount = 0, listArcCount = 0; while(fstEnum.next() != null) { - if (fstEnum.arcs[fstEnum.upto].bytesPerArc == 0) { + if (fstEnum.arcs[fstEnum.upto].bytesPerArc() == 0) { listArcCount ++; - } else if (fstEnum.arcs[fstEnum.upto].arcIdx == Integer.MIN_VALUE) { + } else if (fstEnum.arcs[fstEnum.upto].arcIdx() == Integer.MIN_VALUE) { directArrayArcCount ++; } else { sparseArrayArcCount ++; diff --git a/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java b/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java index 7f43b3f89f0..14b742cbe9a 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java +++ b/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java @@ -277,7 +277,7 @@ public final class IDVersionSegmentTermsEnum extends BaseTermsEnum { arc = arcs[0]; assert arc.isFinal(); - output = arc.output; + output = arc.output(); targetUpto = 0; IDVersionSegmentTermsEnumFrame lastFrame = stack[0]; @@ -303,9 +303,9 @@ public final class IDVersionSegmentTermsEnum extends BaseTermsEnum { //if (arc.label != (target.bytes[target.offset + targetUpto] & 0xFF)) { //System.out.println("FAIL: arc.label=" + (char) arc.label + " targetLabel=" + (char) (target.bytes[target.offset + targetUpto] & 0xFF)); //} - assert arc.label == (target.bytes[target.offset + targetUpto] & 0xFF): "arc.label=" + (char) arc.label + " targetLabel=" + (char) (target.bytes[target.offset + targetUpto] & 0xFF); - if (arc.output != VersionBlockTreeTermsWriter.NO_OUTPUT) { - output = VersionBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output); + assert arc.label() == (target.bytes[target.offset + targetUpto] & 0xFF): "arc.label=" + (char) arc.label() + " targetLabel=" + (char) (target.bytes[target.offset + targetUpto] & 0xFF); + if (arc.output() != VersionBlockTreeTermsWriter.NO_OUTPUT) { + output = VersionBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output()); } if (arc.isFinal()) { lastFrame = stack[1+lastFrame.ord]; @@ -404,19 +404,19 @@ public final class IDVersionSegmentTermsEnum extends BaseTermsEnum { // Empty string prefix must have an output (block) in the index! assert arc.isFinal(); - assert arc.output != null; + assert arc.output() != null; // if (DEBUG) { // System.out.println(" no seek state; push root frame"); // } - output = arc.output; + output = arc.output(); currentFrame = staticFrame; //term.length = 0; targetUpto = 0; - currentFrame = pushFrame(arc, VersionBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput), 0); + currentFrame = pushFrame(arc, VersionBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput()), 0); } // if (DEBUG) { @@ -517,9 +517,9 @@ public final class IDVersionSegmentTermsEnum extends BaseTermsEnum { termExists = false; } // Aggregate output as we go: - assert arc.output != null; - if (arc.output != VersionBlockTreeTermsWriter.NO_OUTPUT) { - output = VersionBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output); + assert arc.output() != null; + if (arc.output() != VersionBlockTreeTermsWriter.NO_OUTPUT) { + output = VersionBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output()); } // if (DEBUG) { @@ -529,7 +529,7 @@ public final class IDVersionSegmentTermsEnum extends BaseTermsEnum { if (arc.isFinal()) { // if (DEBUG) System.out.println(" arc is final!"); - currentFrame = pushFrame(arc, VersionBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput), targetUpto); + currentFrame = pushFrame(arc, VersionBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput()), targetUpto); // if (DEBUG) System.out.println(" curFrame.ord=" + currentFrame.ord + " hasTerms=" + currentFrame.hasTerms); } } @@ -619,7 +619,7 @@ public final class IDVersionSegmentTermsEnum extends BaseTermsEnum { arc = arcs[0]; assert arc.isFinal(); - output = arc.output; + output = arc.output(); targetUpto = 0; IDVersionSegmentTermsEnumFrame lastFrame = stack[0]; @@ -642,14 +642,14 @@ public final class IDVersionSegmentTermsEnum extends BaseTermsEnum { break; } arc = arcs[1+targetUpto]; - assert arc.label == (target.bytes[target.offset + targetUpto] & 0xFF): "arc.label=" + (char) arc.label + " targetLabel=" + (char) (target.bytes[target.offset + targetUpto] & 0xFF); + assert arc.label() == (target.bytes[target.offset + targetUpto] & 0xFF): "arc.label=" + (char) arc.label() + " targetLabel=" + (char) (target.bytes[target.offset + targetUpto] & 0xFF); // TODO: we could save the outputs in local // byte[][] instead of making new objs ever // seek; but, often the FST doesn't have any // shared bytes (but this could change if we // reverse vLong byte order) - if (arc.output != VersionBlockTreeTermsWriter.NO_OUTPUT) { - output = VersionBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output); + if (arc.output() != VersionBlockTreeTermsWriter.NO_OUTPUT) { + output = VersionBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output()); } if (arc.isFinal()) { lastFrame = stack[1+lastFrame.ord]; @@ -722,19 +722,19 @@ public final class IDVersionSegmentTermsEnum extends BaseTermsEnum { // Empty string prefix must have an output (block) in the index! assert arc.isFinal(); - assert arc.output != null; + assert arc.output() != null; //if (DEBUG) { //System.out.println(" no seek state; push root frame"); //} - output = arc.output; + output = arc.output(); currentFrame = staticFrame; //term.length = 0; targetUpto = 0; - currentFrame = pushFrame(arc, VersionBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput), 0); + currentFrame = pushFrame(arc, VersionBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput()), 0); } //if (DEBUG) { @@ -789,9 +789,9 @@ public final class IDVersionSegmentTermsEnum extends BaseTermsEnum { term.setByteAt(targetUpto, (byte) targetLabel); arc = nextArc; // Aggregate output as we go: - assert arc.output != null; - if (arc.output != VersionBlockTreeTermsWriter.NO_OUTPUT) { - output = VersionBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output); + assert arc.output() != null; + if (arc.output() != VersionBlockTreeTermsWriter.NO_OUTPUT) { + output = VersionBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output()); } //if (DEBUG) { @@ -801,7 +801,7 @@ public final class IDVersionSegmentTermsEnum extends BaseTermsEnum { if (arc.isFinal()) { //if (DEBUG) System.out.println(" arc is final!"); - currentFrame = pushFrame(arc, VersionBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput), targetUpto); + currentFrame = pushFrame(arc, VersionBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.nextFinalOutput()), targetUpto); //if (DEBUG) System.out.println(" curFrame.ord=" + currentFrame.ord + " hasTerms=" + currentFrame.hasTerms); } } @@ -854,8 +854,8 @@ public final class IDVersionSegmentTermsEnum extends BaseTermsEnum { } if (fr.index != null) { assert !isSeekFrame || f.arc != null: "isSeekFrame=" + isSeekFrame + " f.arc=" + f.arc; - if (f.prefix > 0 && isSeekFrame && f.arc.label != (term.byteAt(f.prefix-1)&0xFF)) { - out.println(" broken seek state: arc.label=" + (char) f.arc.label + " vs term byte=" + (char) (term.byteAt(f.prefix-1)&0xFF)); + if (f.prefix > 0 && isSeekFrame && f.arc.label() != (term.byteAt(f.prefix-1)&0xFF)) { + out.println(" broken seek state: arc.label=" + (char) f.arc.label() + " vs term byte=" + (char) (term.byteAt(f.prefix-1)&0xFF)); throw new RuntimeException("seek state is broken"); } Pair output = Util.get(fr.index, prefix); diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java index cc11fe124a3..6ca21e7ee3c 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java @@ -727,7 +727,7 @@ public class AnalyzingSuggester extends Lookup implements Accountable { if (fst.findTargetArc(END_BYTE, path.fstNode, scratchArc, bytesReader) != null) { // This node has END_BYTE arc leaving, meaning it's an // "exact" match: - searcher.addStartPaths(scratchArc, fst.outputs.add(path.output, scratchArc.output), false, path.input); + searcher.addStartPaths(scratchArc, fst.outputs.add(path.output, scratchArc.output()), false, path.input); } } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FSTUtil.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FSTUtil.java index 61f2fe12af1..97ef9a62ea1 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FSTUtil.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FSTUtil.java @@ -107,7 +107,7 @@ public class FSTUtil { newInput.append(t.min); queue.add(new Path<>(t.dest, new FST.Arc() .copyFrom(nextArc), fst.outputs - .add(path.output, nextArc.output), newInput)); + .add(path.output, nextArc.output()), newInput)); } } else { // TODO: if this transition's TO state is accepting, and @@ -119,21 +119,21 @@ public class FSTUtil { // done in AnalyzingSuggester). FST.Arc nextArc = Util.readCeilArc(min, fst, path.fstNode, scratchArc, fstReader); - while (nextArc != null && nextArc.label <= max) { - assert nextArc.label <= max; - assert nextArc.label >= min : nextArc.label + " " + while (nextArc != null && nextArc.label() <= max) { + assert nextArc.label() <= max; + assert nextArc.label() >= min : nextArc.label() + " " + min; final IntsRefBuilder newInput = new IntsRefBuilder(); newInput.copyInts(currentInput.get()); - newInput.append(nextArc.label); + newInput.append(nextArc.label()); queue.add(new Path<>(t.dest, new FST.Arc() .copyFrom(nextArc), fst.outputs - .add(path.output, nextArc.output), newInput)); - final int label = nextArc.label; // used in assert + .add(path.output, nextArc.output()), newInput)); + final int label = nextArc.label(); // used in assert nextArc = nextArc.isLast() ? null : fst.readNextRealArc(nextArc, fstReader); - assert nextArc == null || label < nextArc.label : "last: " + label - + " next: " + nextArc.label; + assert nextArc == null || label < nextArc.label() : "last: " + label + + " next: " + nextArc.label(); } } } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java index a16db8a47e7..10e9bb40593 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java @@ -599,7 +599,7 @@ public class FreeTextSuggester extends Lookup implements Accountable { @Override protected void addIfCompetitive(Util.FSTPath path) { - if (path.arc.label != separator) { + if (path.arc.label() != separator) { //System.out.println(" keep path: " + Util.toBytesRef(path.input, new BytesRef()).utf8ToString() + "; " + path + "; arc=" + path.arc); super.addIfCompetitive(path); } else { @@ -718,7 +718,7 @@ public class FreeTextSuggester extends Lookup implements Accountable { if (fst.findTargetArc(bytes[pos++] & 0xff, arc, arc, bytesReader) == null) { return null; } else { - output = fst.outputs.add(output, arc.output); + output = fst.outputs.add(output, arc.output()); } } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java index 6a9e494c3d4..3256ead930e 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java @@ -159,7 +159,7 @@ public final class NRTSuggester implements Accountable { // We are removing dups if (path.payload == -1) { // This path didn't yet see the complete surface form; let's see if it just did with the arc output we just added: - BytesRef arcOutput = path.arc.output.output2; + BytesRef arcOutput = path.arc.output().output2; BytesRef output = path.output.output2; for(int i=0;i= num) return true; } else { diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java index d6c1a97e4fa..cb87492cd09 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java @@ -186,7 +186,7 @@ public class WFSTCompletionLookup extends Lookup implements Accountable { CharsRefBuilder spare = new CharsRefBuilder(); if (exactFirst && arc.isFinal()) { spare.copyUTF8Bytes(scratch.get()); - results.add(new LookupResult(spare.toString(), decodeWeight(prefixOutput + arc.nextFinalOutput))); + results.add(new LookupResult(spare.toString(), decodeWeight(prefixOutput + arc.nextFinalOutput()))); if (--num == 0) { return results; // that was quick } @@ -227,7 +227,7 @@ public class WFSTCompletionLookup extends Lookup implements Accountable { if (fst.findTargetArc(bytes[pos++] & 0xff, arc, arc, bytesReader) == null) { return null; } else { - output += arc.output.longValue(); + output += arc.output().longValue(); } } @@ -250,7 +250,7 @@ public class WFSTCompletionLookup extends Lookup implements Accountable { if (result == null || !arc.isFinal()) { return null; } else { - return Integer.valueOf(decodeWeight(result + arc.nextFinalOutput)); + return Integer.valueOf(decodeWeight(result + arc.nextFinalOutput())); } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java b/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java index 438a6dcd863..590841d8e78 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java @@ -220,7 +220,7 @@ public class FSTTester { return null; } } - output = fst.outputs.add(output, arc.output); + output = fst.outputs.add(output, arc.output()); } if (prefixLength != null) { @@ -253,14 +253,14 @@ public class FSTTester { arcs.clear(); // accumulate output - output = fst.outputs.add(output, arc.output); + output = fst.outputs.add(output, arc.output()); // append label - if (arc.label == FST.END_LABEL) { + if (arc.label() == FST.END_LABEL) { break; } - in.append(arc.label); + in.append(arc.label()); } return output; From fe0c042470dc1a1ba7ffd27f91ac7bc96c3254a0 Mon Sep 17 00:00:00 2001 From: Michael Sokolov Date: Thu, 4 Jul 2019 09:45:51 -0400 Subject: [PATCH 02/38] LUCENE-8920: remove Arc setters, moving implementations into Arc, or copying data into consumers --- .../blocktreeords/OrdsSegmentTermsEnum.java | 8 +- .../codecs/memory/FSTOrdTermsReader.java | 19 +++-- .../lucene/codecs/memory/FSTTermsReader.java | 29 +++---- .../java/org/apache/lucene/util/fst/FST.java | 77 +++++++++++-------- .../org/apache/lucene/util/fst/FSTEnum.java | 34 ++++---- .../java/org/apache/lucene/util/fst/Util.java | 27 ++----- 6 files changed, 90 insertions(+), 104 deletions(-) diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java index 7bfaab55ffa..d6a182b5a73 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java @@ -1115,10 +1115,7 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { } } - if (found) { - // Keep recursing - arc.arcIdx(mid - 1); - } else { + if (found == false) { result.setLength(bestUpto); InputOutput io = new InputOutput(); io.input = result.get(); @@ -1127,9 +1124,8 @@ public final class OrdsSegmentTermsEnum extends BaseTermsEnum { return io; } - fr.index.readNextRealArc(arc, fstReader); - // Recurse on this arc: + fr.index.readArcByIndex(arc, fstReader, mid); result.setIntAt(upto++, arc.label()); output = OrdsBlockTreeTermsWriter.FST_OUTPUTS.add(output, arc.output()); diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java index d653c1209dd..daba6096c93 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java @@ -65,7 +65,7 @@ import org.apache.lucene.util.fst.Util; * FST-based terms dictionary reader. * * The FST index maps each term and its ord, and during seek - * the ord is used fetch metadata from a single block. + * the ord is used to fetch metadata from a single block. * The term dictionary is fully memory resident. * * @lucene.experimental @@ -305,7 +305,7 @@ public class FSTOrdTermsReader extends FieldsProducer { } // Only wraps common operations for PBF interact - abstract class BaseTermsEnum extends org.apache.lucene.index.BaseTermsEnum { + abstract class BaseTermsEnum extends org.apache.lucene.index.BaseTermsEnum { /* Current term's ord, starts from 0 */ long ord; @@ -563,6 +563,8 @@ public class FSTOrdTermsReader extends FieldsProducer { /* fst stats */ FST.Arc arc; + Long output; + /* automaton stats */ int state; @@ -620,9 +622,7 @@ public class FSTOrdTermsReader extends FieldsProducer { @Override void decodeStats() throws IOException { - final FST.Arc arc = topFrame().arc; - assert arc.nextFinalOutput() == fstOutputs.getNoOutput(); - ord = arc.output(); + ord = topFrame().output; super.decodeStats(); } @@ -704,8 +704,7 @@ public class FSTOrdTermsReader extends FieldsProducer { /** Virtual frame, never pop */ Frame loadVirtualFrame(Frame frame) { - frame.arc.output(fstOutputs.getNoOutput()); - frame.arc.nextFinalOutput(fstOutputs.getNoOutput()); + frame.output = fstOutputs.getNoOutput(); frame.state = -1; return frame; } @@ -713,6 +712,7 @@ public class FSTOrdTermsReader extends FieldsProducer { /** Load frame for start arc(node) on fst */ Frame loadFirstFrame(Frame frame) { frame.arc = fst.getFirstArc(frame.arc); + frame.output = frame.arc.output(); frame.state = 0; return frame; } @@ -724,6 +724,7 @@ public class FSTOrdTermsReader extends FieldsProducer { } frame.arc = fst.readFirstRealTargetArc(top.arc.target(), frame.arc, fstReader); frame.state = fsa.step(top.state, frame.arc.label()); + frame.output = frame.arc.output(); //if (TEST) System.out.println(" loadExpand frame="+frame); if (frame.state == -1) { return loadNextFrame(top, frame); @@ -738,6 +739,7 @@ public class FSTOrdTermsReader extends FieldsProducer { } while (!frame.arc.isLast()) { frame.arc = fst.readNextRealArc(frame.arc, fstReader); + frame.output = frame.arc.output(); frame.state = fsa.step(top.state, frame.arc.label()); if (frame.state != -1) { break; @@ -763,6 +765,7 @@ public class FSTOrdTermsReader extends FieldsProducer { if (frame.state == -1) { return loadNextFrame(top, frame); } + frame.output = arc.output(); return frame; } @@ -781,7 +784,7 @@ public class FSTOrdTermsReader extends FieldsProducer { void pushFrame(Frame frame) { final FST.Arc arc = frame.arc; - arc.output(fstOutputs.add(topFrame().arc.output(), arc.output())); + frame.output = fstOutputs.add(topFrame().output, frame.output); term = grow(arc.label()); level++; assert frame == stack[level]; diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java index 41a992fafea..a5abfff355a 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java @@ -438,6 +438,8 @@ public class FSTTermsReader extends FieldsProducer { /* fst stats */ FST.Arc fstArc; + FSTTermOutputs.TermData output; + /* automaton stats */ int fsaState; @@ -464,11 +466,9 @@ public class FSTTermsReader extends FieldsProducer { this.stack[i] = new Frame(); } - Frame frame; - frame = loadVirtualFrame(newFrame()); + loadVirtualFrame(newFrame()); this.level++; - frame = loadFirstFrame(newFrame()); - pushFrame(frame); + pushFrame(loadFirstFrame(newFrame())); this.meta = null; this.metaUpto = 1; @@ -502,18 +502,18 @@ public class FSTTermsReader extends FieldsProducer { /** Lazily accumulate meta data, when we got a accepted term */ void loadMetaData() { - FST.Arc last, next; - last = stack[metaUpto].fstArc; + Frame last, next; + last = stack[metaUpto]; while (metaUpto != level) { metaUpto++; - next = stack[metaUpto].fstArc; - next.output(fstOutputs.add(next.output(), last.output())); + next = stack[metaUpto]; + next.output = fstOutputs.add(next.output, last.output); last = next; } - if (last.isFinal()) { - meta = fstOutputs.add(last.output(), last.nextFinalOutput()); + if (last.fstArc.isFinal()) { + meta = fstOutputs.add(last.output, last.fstArc.nextFinalOutput()); } else { - meta = last.output(); + meta = last.output; } state.docFreq = meta.docFreq; state.totalTermFreq = meta.totalTermFreq; @@ -604,8 +604,7 @@ public class FSTTermsReader extends FieldsProducer { /** Virtual frame, never pop */ Frame loadVirtualFrame(Frame frame) { - frame.fstArc.output(fstOutputs.getNoOutput()); - frame.fstArc.nextFinalOutput(fstOutputs.getNoOutput()); + frame.output = fstOutputs.getNoOutput(); frame.fsaState = -1; return frame; } @@ -613,6 +612,7 @@ public class FSTTermsReader extends FieldsProducer { /** Load frame for start arc(node) on fst */ Frame loadFirstFrame(Frame frame) throws IOException { frame.fstArc = fst.getFirstArc(frame.fstArc); + frame.output = frame.fstArc.output(); frame.fsaState = 0; return frame; } @@ -628,6 +628,7 @@ public class FSTTermsReader extends FieldsProducer { if (frame.fsaState == -1) { return loadNextFrame(top, frame); } + frame.output = frame.fstArc.output(); return frame; } @@ -647,6 +648,7 @@ public class FSTTermsReader extends FieldsProducer { if (frame.fsaState == -1) { return null; } + frame.output = frame.fstArc.output(); return frame; } @@ -663,6 +665,7 @@ public class FSTTermsReader extends FieldsProducer { if (frame.fsaState == -1) { return loadNextFrame(top, frame); } + frame.output = frame.fstArc.output(); return frame; } diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java index f9d0cd51605..cbef9098d08 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java @@ -186,7 +186,7 @@ public final class FST implements Accountable { } boolean flag(int flag) { - return FST.flag(flags(), flag); + return FST.flag(flags, flag); } public boolean isLast() { @@ -230,18 +230,10 @@ public final class FST implements Accountable { return label; } - public void label(int label) { - this.label = label; - } - public T output() { return output; } - public void output(T output) { - this.output = output; - } - /** To node (ord or address) */ public long target() { return target; @@ -251,30 +243,14 @@ public final class FST implements Accountable { return flags; } - public void flags(byte flags) { - this.flags = flags; - } - public T nextFinalOutput() { return nextFinalOutput; } - public void nextFinalOutput(T output) { - nextFinalOutput = output; - } - long nextArc() { return nextArc; } - /** - * Set the position of the next arc to read - * @param nextArc the position to set - */ - public void nextArc(long nextArc) { - this.nextArc = nextArc; - } - /** Where the first arc in the array starts; only valid if * bytesPerArc != 0 */ public long posArcsStart() { @@ -298,14 +274,6 @@ public final class FST implements Accountable { return arcIdx; } - /** - * Set the arcIdx - * @param idx the value to set - */ - public void arcIdx(int idx) { - arcIdx = idx; - } - /** How many arc, if bytesPerArc == 0. Otherwise, the size of the arc array. If the array is * direct, this may include holes. Otherwise it is also how many arcs are in the array */ public int numArcs() { @@ -818,7 +786,7 @@ public final class FST implements Accountable { arc.flags = BIT_FINAL_ARC | BIT_LAST_ARC; arc.nextFinalOutput = emptyOutput; if (emptyOutput != NO_OUTPUT) { - arc.flags |= BIT_ARC_HAS_FINAL_OUTPUT; + arc.flags = (byte) (arc.flags() | BIT_ARC_HAS_FINAL_OUTPUT); } } else { arc.flags = BIT_LAST_ARC; @@ -1033,6 +1001,27 @@ public final class FST implements Accountable { return readLabel(in); } + public Arc readArcAtPosition(Arc arc, final BytesReader in, long pos) throws IOException { + in.setPosition(pos); + arc.flags = in.readByte(); + arc.nextArc = pos; + while (flag(arc.flags(), BIT_MISSING_ARC)) { + // skip empty arcs + arc.nextArc -= arc.bytesPerArc(); + in.skipBytes(arc.bytesPerArc() - 1); + arc.flags = in.readByte(); + } + return readArc(arc, in); + } + + public Arc readArcByIndex(Arc arc, final BytesReader in, int idx) throws IOException { + arc.arcIdx = idx; + assert arc.arcIdx() < arc.numArcs(); + in.setPosition(arc.posArcsStart() - arc.arcIdx() * arc.bytesPerArc()); + arc.flags = in.readByte(); + return readArc(arc, in); + } + /** Never returns null, but you should never call this if * arc.isLast() is true. */ public Arc readNextRealArc(Arc arc, final BytesReader in) throws IOException { @@ -1064,7 +1053,10 @@ public final class FST implements Accountable { in.setPosition(arc.nextArc()); arc.flags = in.readByte(); } + return readArc(arc, in); + } + private Arc readArc(Arc arc, BytesReader in) throws IOException { arc.label = readLabel(in); if (arc.flag(BIT_ARC_HAS_OUTPUT)) { @@ -1118,6 +1110,23 @@ public final class FST implements Accountable { return arc; } + static Arc readEndArc(Arc follow, Arc arc) { + if (follow.isFinal()) { + if (follow.target() <= 0) { + arc.flags = FST.BIT_LAST_ARC; + } else { + arc.flags = 0; + // NOTE: nextArc is a node (not an address!) in this case: + arc.nextArc = follow.target(); + } + arc.output = follow.nextFinalOutput(); + arc.label = FST.END_LABEL; + return arc; + } else { + return null; + } + } + // LUCENE-5152: called only from asserts, to validate that the // non-cached arc lookup would produce the same result, to // catch callers that illegally modify shared structures with diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java index 1c41d31f400..feeddf3fb6b 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java @@ -161,8 +161,7 @@ abstract class FSTEnum { int arcOffset = targetLabel - firstLabel; if (arcOffset >= arc.numArcs()) { // target is beyond the last arc - arc.nextArc(arc.posArcsStart() - (arc.numArcs() - 1) * arc.bytesPerArc()); - fst.readNextRealArc(arc, in); + fst.readArcAtPosition(arc, in, arc.posArcsStart() - (arc.numArcs() - 1) * arc.bytesPerArc()); assert arc.isLast(); // Dead end (target is after the last arc); // rollback to last fork then push @@ -182,12 +181,13 @@ abstract class FSTEnum { } } else { // TODO: if firstLabel == targetLabel + long pos; if (arcOffset >= 0) { - arc.nextArc(arc.posArcsStart() - (arc.bytesPerArc() * arcOffset)); + pos = arc.posArcsStart() - (arc.bytesPerArc() * arcOffset); } else { - arc.nextArc(arc.posArcsStart()); + pos = arc.posArcsStart(); } - fst.readNextRealArc(arc, in); + fst.readArcAtPosition(arc, in, pos); if (arc.label() == targetLabel) { // found -- copy pasta from below output[upto] = fst.outputs.add(output[upto-1], arc.output()); @@ -234,8 +234,7 @@ abstract class FSTEnum { // the outer else clause): if (found) { // Match - arc.arcIdx(mid - 1); - fst.readNextRealArc(arc, in); + fst.readArcByIndex(arc, in, mid); assert arc.arcIdx() == mid; assert arc.label() == targetLabel: "arc.label=" + arc.label() + " vs targetLabel=" + targetLabel + " mid=" + mid; output[upto] = fst.outputs.add(output[upto-1], arc.output()); @@ -247,8 +246,7 @@ abstract class FSTEnum { return fst.readFirstTargetArc(arc, getArc(upto), fstReader); } else if (low == arc.numArcs()) { // Dead end - arc.arcIdx(arc.numArcs() - 2); - fst.readNextRealArc(arc, in); + fst.readArcByIndex(arc, in, arc.numArcs() - 1); assert arc.isLast(); // Dead end (target is after the last arc); // rollback to last fork then push @@ -267,8 +265,7 @@ abstract class FSTEnum { upto--; } } else { - arc.arcIdx(low - 1); - fst.readNextRealArc(arc, in); + fst.readArcByIndex(arc, in, low); assert arc.label() > targetLabel; pushFirst(); return null; @@ -386,15 +383,13 @@ abstract class FSTEnum { } } else { if (targetOffset >= arc.numArcs()) { - arc.nextArc(arc.posArcsStart() - arc.bytesPerArc() * (arc.numArcs() - 1)); - fst.readNextRealArc(arc, in); + fst.readArcAtPosition(arc, in, arc.posArcsStart() - arc.bytesPerArc() * (arc.numArcs() - 1)); assert arc.isLast(); assert arc.label() < targetLabel: "arc.label=" + arc.label() + " vs targetLabel=" + targetLabel; pushLast(); return null; } - arc.nextArc(arc.posArcsStart() - arc.bytesPerArc() * targetOffset); - fst.readNextRealArc(arc, in); + fst.readArcAtPosition(arc, in, arc.posArcsStart() - arc.bytesPerArc() * targetOffset); if (arc.label() == targetLabel) { // found -- copy pasta from below output[upto] = fst.outputs.add(output[upto-1], arc.output()); @@ -408,8 +403,7 @@ abstract class FSTEnum { // Scan backwards to find a floor arc that is not missing for (long arcOffset = arc.posArcsStart() - targetOffset * arc.bytesPerArc(); arcOffset <= arc.posArcsStart(); arcOffset += arc.bytesPerArc()) { // TODO: we can do better here by skipping missing arcs - arc.nextArc(arcOffset); - fst.readNextRealArc(arc, in); + fst.readArcAtPosition(arc, in, arcOffset); if (arc.label() < targetLabel) { assert arc.isLast() || fst.readNextArcLabel(arc, in) > targetLabel; pushLast(); @@ -451,8 +445,7 @@ abstract class FSTEnum { if (found) { // Match -- recurse //System.out.println(" match! arcIdx=" + mid); - arc.arcIdx(mid - 1); - fst.readNextRealArc(arc, in); + fst.readArcByIndex(arc, in, mid); assert arc.arcIdx() == mid; assert arc.label() == targetLabel: "arc.label=" + arc.label() + " vs targetLabel=" + targetLabel + " mid=" + mid; output[upto] = fst.outputs.add(output[upto-1], arc.output()); @@ -491,8 +484,7 @@ abstract class FSTEnum { } } else { // There is a floor arc: - arc.arcIdx(high - 1); - fst.readNextRealArc(arc, in); + fst.readArcByIndex(arc, in, high); assert arc.isLast() || fst.readNextArcLabel(arc, in) > targetLabel; assert arc.label() < targetLabel: "arc.label=" + arc.label() + " vs targetLabel=" + targetLabel; pushLast(); diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/Util.java b/lucene/core/src/java/org/apache/lucene/util/fst/Util.java index 0b2a0eea503..e033267ffbe 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/Util.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/Util.java @@ -191,8 +191,7 @@ public final class Util { idx = low - 1; } - arc.arcIdx(idx - 1); - fst.readNextRealArc(arc, in); + fst.readArcByIndex(arc, in, idx); result.setIntAt(upto++, arc.label()); output += arc.output(); @@ -934,20 +933,7 @@ public final class Util { */ public static Arc readCeilArc(int label, FST fst, Arc follow, Arc arc, BytesReader in) throws IOException { if (label == FST.END_LABEL) { - if (follow.isFinal()) { - if (follow.target() <= 0) { - arc.flags((byte) FST.BIT_LAST_ARC); - } else { - arc.flags((byte) 0); - // NOTE: nextArc is a node (not an address!) in this case: - arc.nextArc(follow.target()); - } - arc.output(follow.nextFinalOutput()); - arc.label(FST.END_LABEL); - return arc; - } else { - return null; - } + return FST.readEndArc(follow, arc); } if (!FST.targetHasArcs(follow)) { return null; @@ -962,8 +948,7 @@ public final class Util { } else if (offset < 0) { return arc; } else { - arc.nextArc(arc.posArcsStart() - offset * arc.bytesPerArc()); - return fst.readNextRealArc(arc, in); + return fst.readArcAtPosition(arc, in, arc.posArcsStart() - offset * arc.bytesPerArc()); } } // Arcs are packed array -- use binary search to find @@ -987,16 +972,14 @@ public final class Util { } else if (cmp > 0) { high = mid - 1; } else { - arc.arcIdx(mid - 1); - return fst.readNextRealArc(arc, in); + return fst.readArcByIndex(arc, in, mid); } } if (low == arc.numArcs()) { // DEAD END! return null; } - arc.arcIdx(high + 1); - return fst.readNextRealArc(arc, in ); + return fst.readArcByIndex(arc, in , high + 1); } // Linear scan From 92d4e712d5d50d745c5a6c10dacda66198974116 Mon Sep 17 00:00:00 2001 From: Michael Sokolov Date: Thu, 4 Jul 2019 10:45:17 -0400 Subject: [PATCH 03/38] LUCENE-8920: refactor FST binary search --- .../codecs/memory/FSTOrdTermsReader.java | 2 +- .../lucene/util/fst/BytesRefFSTEnum.java | 2 +- .../org/apache/lucene/util/fst/FSTEnum.java | 89 ++++---------- .../lucene/util/fst/IntsRefFSTEnum.java | 2 +- .../java/org/apache/lucene/util/fst/Util.java | 64 ++++++---- .../org/apache/lucene/util/fst/TestUtil.java | 115 ++++++++++++++++++ 6 files changed, 180 insertions(+), 94 deletions(-) create mode 100644 lucene/core/src/test/org/apache/lucene/util/fst/TestUtil.java diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java index daba6096c93..0fa8ebe0524 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java @@ -305,7 +305,7 @@ public class FSTOrdTermsReader extends FieldsProducer { } // Only wraps common operations for PBF interact - abstract class BaseTermsEnum extends org.apache.lucene.index.BaseTermsEnum { + abstract class BaseTermsEnum extends org.apache.lucene.index.BaseTermsEnum { /* Current term's ord, starts from 0 */ long ord; diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/BytesRefFSTEnum.java b/lucene/core/src/java/org/apache/lucene/util/fst/BytesRefFSTEnum.java index 97d6a0eb357..19e2a01c575 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/BytesRefFSTEnum.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/BytesRefFSTEnum.java @@ -81,7 +81,7 @@ public final class BytesRefFSTEnum extends FSTEnum { public InputOutput seekExact(BytesRef target) throws IOException { this.target = target; targetLength = target.length; - if (super.doSeekExact()) { + if (doSeekExact()) { assert upto == 1+target.length; return setResult(); } else { diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java index feeddf3fb6b..36d8ddd384a 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java @@ -24,7 +24,7 @@ import org.apache.lucene.util.RamUsageEstimator; /** Can next() and advance() through the terms in an FST * - * @lucene.experimental + * @lucene.experimental */ abstract class FSTEnum { @@ -207,36 +207,12 @@ abstract class FSTEnum { private FST.Arc doSeekCeilArrayPacked(final FST.Arc arc, final int targetLabel, final FST.BytesReader in) throws IOException { // The array is packed -- use binary search to find the target. - - int low = arc.arcIdx(); - int high = arc.numArcs() -1; - int mid = 0; - //System.out.println("do arc array low=" + low + " high=" + high + " targetLabel=" + targetLabel); - boolean found = false; - while (low <= high) { - mid = (low + high) >>> 1; - in.setPosition(arc.posArcsStart()); - in.skipBytes(arc.bytesPerArc() * mid + 1); - final int midLabel = fst.readLabel(in); - final int cmp = midLabel - targetLabel; - //System.out.println(" cycle low=" + low + " high=" + high + " mid=" + mid + " midLabel=" + midLabel + " cmp=" + cmp); - if (cmp < 0) - low = mid + 1; - else if (cmp > 0) - high = mid - 1; - else { - found = true; - break; - } - } - - // NOTE: this code is dup'd w/ the code below (in - // the outer else clause): - if (found) { + int idx = Util.binarySearch(fst, arc, targetLabel); + if (idx >= 0) { // Match - fst.readArcByIndex(arc, in, mid); - assert arc.arcIdx() == mid; - assert arc.label() == targetLabel: "arc.label=" + arc.label() + " vs targetLabel=" + targetLabel + " mid=" + mid; + fst.readArcByIndex(arc, in, idx); + assert arc.arcIdx() == idx; + assert arc.label() == targetLabel: "arc.label=" + arc.label() + " vs targetLabel=" + targetLabel + " mid=" + idx; output[upto] = fst.outputs.add(output[upto-1], arc.output()); if (targetLabel == FST.END_LABEL) { return null; @@ -244,9 +220,11 @@ abstract class FSTEnum { setCurrentLabel(arc.label()); incr(); return fst.readFirstTargetArc(arc, getArc(upto), fstReader); - } else if (low == arc.numArcs()) { + } + idx = -1 - idx; + if (idx == arc.numArcs()) { // Dead end - fst.readArcByIndex(arc, in, arc.numArcs() - 1); + fst.readArcByIndex(arc, in, idx - 1); assert arc.isLast(); // Dead end (target is after the last arc); // rollback to last fork then push @@ -265,7 +243,8 @@ abstract class FSTEnum { upto--; } } else { - fst.readArcByIndex(arc, in, low); + // Ceiling - arc with least higher label + fst.readArcByIndex(arc, in, idx); assert arc.label() > targetLabel; pushFirst(); return null; @@ -314,7 +293,7 @@ abstract class FSTEnum { // Todo: should we return a status here (SEEK_FOUND / SEEK_NOT_FOUND / // SEEK_END)? saves the eq check above? /** Seeks to largest term that's <= target. */ - protected void doSeekFloor() throws IOException { + void doSeekFloor() throws IOException { // TODO: possibly caller could/should provide common // prefix length? ie this work may be redundant if @@ -417,37 +396,14 @@ abstract class FSTEnum { private FST.Arc doSeekFloorArrayPacked(FST.Arc arc, int targetLabel, final FST.BytesReader in) throws IOException { // Arcs are fixed array -- use binary search to find the target. + int idx = Util.binarySearch(fst, arc, targetLabel); - int low = arc.arcIdx(); - int high = arc.numArcs() -1; - int mid = 0; - //System.out.println("do arc array low=" + low + " high=" + high + " targetLabel=" + targetLabel); - boolean found = false; - while (low <= high) { - mid = (low + high) >>> 1; - in.setPosition(arc.posArcsStart()); - in.skipBytes(arc.bytesPerArc() * mid + 1); - final int midLabel = fst.readLabel(in); - final int cmp = midLabel - targetLabel; - //System.out.println(" cycle low=" + low + " high=" + high + " mid=" + mid + " midLabel=" + midLabel + " cmp=" + cmp); - if (cmp < 0) { - low = mid + 1; - } else if (cmp > 0) { - high = mid - 1; - } else { - found = true; - break; - } - } - - // NOTE: this code is dup'd w/ the code below (in - // the outer else clause): - if (found) { + if (idx >= 0) { // Match -- recurse - //System.out.println(" match! arcIdx=" + mid); - fst.readArcByIndex(arc, in, mid); - assert arc.arcIdx() == mid; - assert arc.label() == targetLabel: "arc.label=" + arc.label() + " vs targetLabel=" + targetLabel + " mid=" + mid; + //System.out.println(" match! arcIdx=" + idx); + fst.readArcByIndex(arc, in, idx); + assert arc.arcIdx() == idx; + assert arc.label() == targetLabel: "arc.label=" + arc.label() + " vs targetLabel=" + targetLabel + " mid=" + idx; output[upto] = fst.outputs.add(output[upto-1], arc.output()); if (targetLabel == FST.END_LABEL) { return null; @@ -455,7 +411,7 @@ abstract class FSTEnum { setCurrentLabel(arc.label()); incr(); return fst.readFirstTargetArc(arc, getArc(upto), fstReader); - } else if (high == -1) { + } else if (idx == -1) { //System.out.println(" before first"); // Very first arc is after our target // TODO: if each arc could somehow read the arc just @@ -483,8 +439,8 @@ abstract class FSTEnum { arc = getArc(upto); } } else { - // There is a floor arc: - fst.readArcByIndex(arc, in, high); + // There is a floor arc; idx will be {@code -1 - (floor + 1)}. + fst.readArcByIndex(arc, in, -2 - idx); assert arc.isLast() || fst.readNextArcLabel(arc, in) > targetLabel; assert arc.label() < targetLabel: "arc.label=" + arc.label() + " vs targetLabel=" + targetLabel; pushLast(); @@ -652,4 +608,5 @@ abstract class FSTEnum { } return arcs[idx]; } + } diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/IntsRefFSTEnum.java b/lucene/core/src/java/org/apache/lucene/util/fst/IntsRefFSTEnum.java index f485854a748..2c05c965fbd 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/IntsRefFSTEnum.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/IntsRefFSTEnum.java @@ -81,7 +81,7 @@ public final class IntsRefFSTEnum extends FSTEnum { public InputOutput seekExact(IntsRef target) throws IOException { this.target = target; targetLength = target.length; - if (super.doSeekExact()) { + if (doSeekExact()) { assert upto == 1+target.length; return setResult(); } else { diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/Util.java b/lucene/core/src/java/org/apache/lucene/util/fst/Util.java index e033267ffbe..ddecaded731 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/Util.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/Util.java @@ -951,35 +951,17 @@ public final class Util { return fst.readArcAtPosition(arc, in, arc.posArcsStart() - offset * arc.bytesPerArc()); } } - // Arcs are packed array -- use binary search to find - // the target. - - int low = arc.arcIdx(); - int mid = 0; - int high = arc.numArcs() - 1; - // System.out.println("do arc array low=" + low + " high=" + high + - // " targetLabel=" + targetLabel); - while (low <= high) { - mid = (low + high) >>> 1; - in.setPosition(arc.posArcsStart()); - in.skipBytes(arc.bytesPerArc() * mid + 1); - final int midLabel = fst.readLabel(in); - final int cmp = midLabel - label; - // System.out.println(" cycle low=" + low + " high=" + high + " mid=" + - // mid + " midLabel=" + midLabel + " cmp=" + cmp); - if (cmp < 0) { - low = mid + 1; - } else if (cmp > 0) { - high = mid - 1; - } else { - return fst.readArcByIndex(arc, in, mid); - } + // Arcs are packed array -- use binary search to find the target. + int idx = binarySearch(fst, arc, label); + if (idx >= 0) { + return fst.readArcByIndex(arc, in, idx); } - if (low == arc.numArcs()) { + idx = -1 - idx; + if (idx == arc.numArcs()) { // DEAD END! return null; } - return fst.readArcByIndex(arc, in , high + 1); + return fst.readArcByIndex(arc, in , idx); } // Linear scan @@ -1001,4 +983,36 @@ public final class Util { } } + /** + * Perform a binary search of Arcs encoded as a packed array + * @param fst the FST from which to read + * @param arc the starting arc; sibling arcs greater than this will be searched. Usually the first arc in the array. + * @param targetLabel the label to search for + * @param the output type of the FST + * @return the index of the Arc having the target label, or if no Arc has the matching label, {@code -1 - idx)}, + * where {@code idx} is the index of the Arc with the next highest label, or the total number of arcs + * if the target label exceeds the maximum. + * @throws IOException when the FST reader does + */ + static int binarySearch(FST fst, FST.Arc arc, int targetLabel) throws IOException { + BytesReader in = fst.getBytesReader(); + int low = arc.arcIdx(); + int mid = 0; + int high = arc.numArcs() -1; + while (low <= high) { + mid = (low + high) >>> 1; + in.setPosition(arc.posArcsStart()); + in.skipBytes(arc.bytesPerArc() * mid + 1); + final int midLabel = fst.readLabel(in); + final int cmp = midLabel - targetLabel; + if (cmp < 0) { + low = mid + 1; + } else if (cmp > 0) { + high = mid - 1; + } else { + return mid; + } + } + return -1 - low; + } } diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/TestUtil.java b/lucene/core/src/test/org/apache/lucene/util/fst/TestUtil.java new file mode 100644 index 00000000000..5ec163e9cfe --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/util/fst/TestUtil.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.util.fst; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IntsRefBuilder; +import org.apache.lucene.util.LuceneTestCase; + +public class TestUtil extends LuceneTestCase { + + public void testBinarySearch() throws Exception { + // Creates a node with 8 arcs spanning (z-A) = 57 chars that will be encoded as a sparse array (no gaps) + // requiring binary search + List letters = Arrays.asList("A", "E", "J", "K", "L", "O", "T", "z"); + FST fst = buildFST(letters, true); + FST.Arc arc = fst.getFirstArc(new FST.Arc<>()); + arc = fst.readFirstTargetArc(arc, arc, fst.getBytesReader()); + for (int i = 0; i < letters.size(); i++) { + assertEquals(i, Util.binarySearch(fst, arc, letters.get(i).charAt(0))); + } + // before the first + assertEquals(-1, Util.binarySearch(fst, arc, ' ')); + // after the last + assertEquals(-1 - letters.size(), Util.binarySearch(fst, arc, '~')); + assertEquals(-2, Util.binarySearch(fst, arc, 'B')); + assertEquals(-2, Util.binarySearch(fst, arc, 'C')); + assertEquals(-7, Util.binarySearch(fst, arc, 'P')); + } + + public void testReadCeilArcPackedArray() throws Exception { + List letters = Arrays.asList("A", "E", "J", "K", "L", "O", "T", "z"); + verifyReadCeilArc(letters, true); + } + + public void testReadCeilArcArrayWithGaps() throws Exception { + List letters = Arrays.asList("A", "E", "J", "K", "L", "O", "T"); + verifyReadCeilArc(letters, true); + } + + public void testReadCeilArcList() throws Exception { + List letters = Arrays.asList("A", "E", "J", "K", "L", "O", "T", "z"); + verifyReadCeilArc(letters, false); + } + + private void verifyReadCeilArc(List letters, boolean allowArrayArcs) throws Exception { + FST fst = buildFST(letters, allowArrayArcs); + FST.Arc first = fst.getFirstArc(new FST.Arc<>()); + FST.Arc arc = new FST.Arc<>(); + FST.BytesReader in = fst.getBytesReader(); + for (String letter : letters) { + char c = letter.charAt(0); + arc = Util.readCeilArc(c, fst, first, arc, in); + assertNotNull(arc); + assertEquals(c, arc.label()); + } + // before the first + assertEquals('A', Util.readCeilArc(' ', fst, first, arc, in).label()); + // after the last + assertNull(Util.readCeilArc('~', fst, first, arc, in)); + // in the middle + assertEquals('J', Util.readCeilArc('F', fst, first, arc, in).label()); + // no following arcs + assertNull(Util.readCeilArc('Z', fst, arc, arc, in)); + } + + private FST buildFST(List words, boolean allowArrayArcs) throws Exception { + final Outputs outputs = NoOutputs.getSingleton(); + final Builder b = new Builder<>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, allowArrayArcs, 15); + + for (String word : words) { + b.add(Util.toIntsRef(new BytesRef(word), new IntsRefBuilder()), outputs.getNoOutput()); + } + return b.finish(); + } + + private List createRandomDictionary(int width, int depth) { + return createRandomDictionary(new ArrayList<>(), new StringBuilder(), width, depth); + } + + private List createRandomDictionary(List dict, StringBuilder buf, int width, int depth) { + char c = (char) random().nextInt(128); + assert width < Character.MIN_SURROGATE / 8 - 128; // avoid surrogate chars + int len = buf.length(); + for (int i = 0; i < width; i++) { + buf.append(c); + if (depth > 0) { + createRandomDictionary(dict, buf, width, depth - 1); + } else { + dict.add(buf.toString()); + } + c += random().nextInt(8); + buf.setLength(len); + } + return dict; + } + +} From 4050ddc59beeff2be5a862782579ceb8e5775c60 Mon Sep 17 00:00:00 2001 From: Chris Hostetter Date: Fri, 26 Jul 2019 18:41:06 -0700 Subject: [PATCH 04/38] Harden RulesTest * ensure all collections/replicas are active * use waitForState or waitForActiveCollection before checking rules/snitch to prevent false failures on stale state * ensure cluster policy is cleared after each test method Some of these changes should also help ensure we don't get (more) spurious failures due to SOLR-13616 --- .../org/apache/solr/cloud/rule/RulesTest.java | 172 ++++++++++++++---- 1 file changed, 138 insertions(+), 34 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java b/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java index 9e4ae27fda9..61dba0a2a32 100644 --- a/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud.rule; import java.lang.invoke.MethodHandles; +import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.util.List; import java.util.Map; @@ -32,6 +33,9 @@ import org.apache.solr.client.solrj.response.SimpleSolrResponse; import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.ModifiableSolrParams; import org.junit.After; import org.junit.BeforeClass; @@ -63,6 +67,9 @@ public class RulesTest extends SolrCloudTestCase { @After public void removeCollections() throws Exception { cluster.deleteAllCollections(); + // clear any cluster policy test methods may have set + cluster.getSolrClient().getZkStateReader().getZkClient().setData(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, + "{}".getBytes(StandardCharsets.UTF_8), true); } @Test @@ -76,7 +83,9 @@ public class RulesTest extends SolrCloudTestCase { .setRule("cores:<4", "node:*,replica:<2", "freedisk:>"+minGB) .setSnitch("class:ImplicitSnitch") .process(cluster.getSolrClient()); - + + cluster.waitForActiveCollection(rulesColl, 1, 2); + DocCollection rulesCollection = getCollectionState(rulesColl); List list = (List) rulesCollection.get("rule"); @@ -91,6 +100,39 @@ public class RulesTest extends SolrCloudTestCase { CollectionAdminRequest.createShard(rulesColl, "shard2").process(cluster.getSolrClient()); CollectionAdminRequest.addReplicaToShard(rulesColl, "shard2").process(cluster.getSolrClient()); + waitForState("Should have found shard1 w/2active replicas + shard2 w/1active replica", + rulesColl, (liveNodes, collection) -> { + // short circut if collection is deleted + // or we don't yet have the correct number of slices + if (null == collection || 2 != collection.getSlices().size()) { + return false; + } + for (Slice slice : collection.getSlices()) { + // short circut if our slice isn't active + if (Slice.State.ACTIVE != slice.getState()) { + return false; + } + if (slice.getName().equals("shard1")) { + // for shard1, we should have 2 fully live replicas + final List liveReplicas = slice.getReplicas + ((r) -> r.isActive(liveNodes)); + if (2 != liveReplicas.size()) { + return false; + } + } else if (slice.getName().equals("shard2")) { + // for shard2, we should have 1 fully live replicas + final List liveReplicas = slice.getReplicas + ((r) -> r.isActive(liveNodes)); + if (1 != liveReplicas.size()) { + return false; + } + } else { + // WTF? + return false; + } + } + return true; + }); } @Test @@ -114,18 +156,37 @@ public class RulesTest extends SolrCloudTestCase { .setRule("port:" + port) .setSnitch("class:ImplicitSnitch") .process(cluster.getSolrClient()); - - // now we assert that the replica placement rule is used instead of the cluster policy - DocCollection rulesCollection = getCollectionState(rulesColl); - List list = (List) rulesCollection.get("rule"); - assertEquals(1, list.size()); - assertEquals(port, ((Map) list.get(0)).get("port")); - list = (List) rulesCollection.get("snitch"); - assertEquals(1, list.size()); - assertEquals ( "ImplicitSnitch", ((Map)list.get(0)).get("class")); - - boolean allOnExpectedPort = rulesCollection.getReplicas().stream().allMatch(replica -> replica.getNodeName().contains(port)); - assertTrue("Not all replicas were found to be on port: " + port + ". Collection state is: " + rulesCollection, allOnExpectedPort); + + waitForState("Collection should have followed port rule w/ImplicitSnitch, not cluster policy", + rulesColl, (liveNodes, rulesCollection) -> { + // first sanity check that the collection exists & the rules/snitch are listed + if (null == rulesCollection) { + return false; + } else { + List list = (List) rulesCollection.get("rule"); + if (null == list || 1 != list.size()) { + return false; + } + if (! port.equals(((Map) list.get(0)).get("port"))) { + return false; + } + list = (List) rulesCollection.get("snitch"); + if (null == list || 1 != list.size()) { + return false; + } + if (! "ImplicitSnitch".equals(((Map)list.get(0)).get("class"))) { + return false; + } + } + if (2 != rulesCollection.getReplicas().size()) { + return false; + } + // now sanity check that the rules were *obeyed* + // (and the contradictory policy was ignored) + return rulesCollection.getReplicas().stream().allMatch + (replica -> (replica.getNodeName().contains(port) && + replica.isActive(liveNodes))); + }); } @Test @@ -140,15 +201,35 @@ public class RulesTest extends SolrCloudTestCase { .setSnitch("class:ImplicitSnitch") .process(cluster.getSolrClient()); - DocCollection rulesCollection = getCollectionState(rulesColl); - - List list = (List) rulesCollection.get("rule"); - assertEquals(1, list.size()); - assertEquals(port, ((Map) list.get(0)).get("port")); - list = (List) rulesCollection.get("snitch"); - assertEquals(1, list.size()); - assertEquals ( "ImplicitSnitch", ((Map)list.get(0)).get("class")); - + waitForState("Collection should have followed port rule w/ImplicitSnitch, not cluster policy", + rulesColl, (liveNodes, rulesCollection) -> { + // first sanity check that the collection exists & the rules/snitch are listed + if (null == rulesCollection) { + return false; + } else { + List list = (List) rulesCollection.get("rule"); + if (null == list || 1 != list.size()) { + return false; + } + if (! port.equals(((Map) list.get(0)).get("port"))) { + return false; + } + list = (List) rulesCollection.get("snitch"); + if (null == list || 1 != list.size()) { + return false; + } + if (! "ImplicitSnitch".equals(((Map)list.get(0)).get("class"))) { + return false; + } + } + if (2 != rulesCollection.getReplicas().size()) { + return false; + } + // now sanity check that the rules were *obeyed* + return rulesCollection.getReplicas().stream().allMatch + (replica -> (replica.getNodeName().contains(port) && + replica.isActive(liveNodes))); + }); } @Test @@ -166,6 +247,8 @@ public class RulesTest extends SolrCloudTestCase { .setRule("ip_2:" + ip_2, "ip_1:" + ip_1) .setSnitch("class:ImplicitSnitch") .process(cluster.getSolrClient()); + + cluster.waitForActiveCollection(rulesColl, 1, 2); DocCollection rulesCollection = getCollectionState(rulesColl); List list = (List) rulesCollection.get("rule"); @@ -232,6 +315,8 @@ public class RulesTest extends SolrCloudTestCase { .setRule("cores:<4", "node:*,replica:1", "freedisk:>" + minGB1) .setSnitch("class:ImplicitSnitch") .process(cluster.getSolrClient()); + + cluster.waitForActiveCollection(rulesColl, 1, 2); // TODO: Make a MODIFYCOLLECTION SolrJ class @@ -244,17 +329,36 @@ public class RulesTest extends SolrCloudTestCase { p.add("autoAddReplicas", "true"); cluster.getSolrClient().request(new GenericSolrRequest(POST, COLLECTIONS_HANDLER_PATH, p)); - DocCollection rulesCollection = getCollectionState(rulesColl); - log.info("version_of_coll {} ", rulesCollection.getZNodeVersion()); - List list = (List) rulesCollection.get("rule"); - assertEquals(3, list.size()); - assertEquals("<5", ((Map) list.get(0)).get("cores")); - assertEquals("1", ((Map) list.get(1)).get("replica")); - assertEquals(">"+minGB2, ((Map) list.get(2)).get("freedisk")); - assertEquals("true", String.valueOf(rulesCollection.getProperties().get("autoAddReplicas"))); - list = (List) rulesCollection.get("snitch"); - assertEquals(1, list.size()); - assertEquals("ImplicitSnitch", ((Map) list.get(0)).get("class")); - + waitForState("Should have found updated rules in DocCollection", + rulesColl, (liveNodes, rulesCollection) -> { + if (null == rulesCollection) { + return false; + } + List list = (List) rulesCollection.get("rule"); + if (null == list || 3 != list.size()) { + return false; + } + if (! "<5".equals(((Map) list.get(0)).get("cores"))) { + return false; + } + if (! "1".equals(((Map) list.get(1)).get("replica"))) { + return false; + } + if (! (">"+minGB2).equals(((Map) list.get(2)).get("freedisk"))) { + return false; + } + if (! "true".equals(String.valueOf(rulesCollection.getProperties().get("autoAddReplicas")))) { + return false; + } + list = (List) rulesCollection.get("snitch"); + if (null == list || 1 != list.size()) { + return false; + } + if (! "ImplicitSnitch".equals(((Map) list.get(0)).get("class"))) { + return false; + } + return true; + }); + } } From 8c8d8abddc9f5f8c92943e50d6169882e7188c44 Mon Sep 17 00:00:00 2001 From: vinod kumar Date: Sun, 28 Jul 2019 23:36:56 +0900 Subject: [PATCH 05/38] LUCENE-8936: Add SpanishMinimalStemFilter Signed-off-by: Tomoko Uchida --- lucene/CHANGES.txt | 4 + .../analysis/es/SpanishMinimalStemFilter.java | 58 ++++++++++++++ .../es/SpanishMinimalStemFilterFactory.java | 52 ++++++++++++ .../analysis/es/SpanishMinimalStemmer.java | 80 +++++++++++++++++++ ...he.lucene.analysis.util.TokenFilterFactory | 1 + .../es/TestSpanishMinimalStemFilter.java | 80 +++++++++++++++++++ .../TestSpanishMinimalStemFilterFactory.java | 47 +++++++++++ 7 files changed, 322 insertions(+) create mode 100644 lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishMinimalStemFilter.java create mode 100644 lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishMinimalStemFilterFactory.java create mode 100644 lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishMinimalStemmer.java create mode 100644 lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishMinimalStemFilter.java create mode 100644 lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishMinimalStemFilterFactory.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 6c1e2a21016..79b1b5ce728 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -56,6 +56,10 @@ API Changes * LUCENE-8909: IndexWriter#getFieldNames() method is used to get fields present in index. After LUCENE-8316, this method is no longer required. Hence, deprecate IndexWriter#getFieldNames() method. (Adrien Grand, Munendra S N) +New Features + +* LUCENE-8936: Add SpanishMinimalStemFilter (vinod kumar via Tomoko Uchida) + Improvements * LUCENE-8874: Show SPI names instead of class names in Luke Analysis tab. (Tomoko Uchida) diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishMinimalStemFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishMinimalStemFilter.java new file mode 100644 index 00000000000..25969a97665 --- /dev/null +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishMinimalStemFilter.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.analysis.es; + +import java.io.IOException; + +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.KeywordAttribute; + +/** + * A {@link TokenFilter} that applies {@link SpanishMinimalStemmer} to stem Spanish + * words. + *

+ * To prevent terms from being stemmed use an instance of + * {@link SetKeywordMarkerFilter} or a custom {@link TokenFilter} that sets + * the {@link KeywordAttribute} before this {@link TokenStream}. + *

+ */ +public final class SpanishMinimalStemFilter extends TokenFilter { + private final SpanishMinimalStemmer stemmer = new SpanishMinimalStemmer(); + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class); + + public SpanishMinimalStemFilter(TokenStream input) { + super(input); + } + + @Override + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + if (!keywordAttr.isKeyword()) { + final int newlen = stemmer.stem(termAtt.buffer(), termAtt.length()); + termAtt.setLength(newlen); + } + return true; + } else { + return false; + } + } +} diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishMinimalStemFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishMinimalStemFilterFactory.java new file mode 100644 index 00000000000..691aa33a537 --- /dev/null +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishMinimalStemFilterFactory.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.analysis.es; + +import java.util.Map; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.util.TokenFilterFactory; + +/** + * Factory for {@link SpanishMinimalStemFilter}. + * 
+ * <fieldType name="text_eslgtstem" class="solr.TextField" positionIncrementGap="100">
+ *   <analyzer>
+ *     <tokenizer class="solr.StandardTokenizerFactory"/>
+ *     <filter class="solr.LowerCaseFilterFactory"/>
+ *     <filter class="solr.SpanishMinimalStemFilterFactory"/>
+ *   </analyzer>
+ * </fieldType>
+ * @lucene.spi {@value #NAME} + */ +public class SpanishMinimalStemFilterFactory extends TokenFilterFactory { + + /** SPI name */ + public static final String NAME = "spanishMinimalStem"; + + /** Creates a new SpanishMinimalStemFilterFactory */ + public SpanishMinimalStemFilterFactory(Map args) { + super(args); + if (!args.isEmpty()) { + throw new IllegalArgumentException("Unknown parameters: " + args); + } + } + + @Override + public TokenStream create(TokenStream input) { return new SpanishMinimalStemFilter(input); } +} diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishMinimalStemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishMinimalStemmer.java new file mode 100644 index 00000000000..b0e3aaec8e9 --- /dev/null +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishMinimalStemmer.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.analysis.es; + +/** + * Minimal plural stemmer for Spanish. + *

+ * This stemmer implements the "plurals" stemmer for + * spanish lanugage. + * + */ +public class SpanishMinimalStemmer { + + public int stem(char s[], int len) { + if (len < 4 || s[len-1] != 's') + return len; + + for (int i = 0; i < len; i++) + switch(s[i]) { + case 'à': + case 'á': + case 'â': + case 'ä': s[i] = 'a'; break; + case 'ò': + case 'ó': + case 'ô': + case 'ö': s[i] = 'o'; break; + case 'è': + case 'é': + case 'ê': + case 'ë': s[i] = 'e'; break; + case 'ù': + case 'ú': + case 'û': + case 'ü': s[i] = 'u'; break; + case 'ì': + case 'í': + case 'î': + case 'ï': s[i] = 'i'; break; + case 'ñ': s[i] = 'n'; break; + } + + switch(s[len-1]) { + case 's': + if (s[len-2] == 'a' || s[len-2] == 'o') { + return len-1; + } + if (s[len-2] == 'e') { + if (s[len-3] == 's' && s[len-4] == 'e') { + return len-2; + } + if (s[len-3] == 'c') { + s[len-3] = 'z'; + return len-2; + } else { + return len-2; + } + } else { + return len-1; + } + } + + return len; + } +} diff --git a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory index 12c8b80d7be..16fca20f84e 100644 --- a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory +++ b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory @@ -45,6 +45,7 @@ org.apache.lucene.analysis.en.EnglishPossessiveFilterFactory org.apache.lucene.analysis.en.KStemFilterFactory org.apache.lucene.analysis.en.PorterStemFilterFactory org.apache.lucene.analysis.es.SpanishLightStemFilterFactory +org.apache.lucene.analysis.es.SpanishMinimalStemFilterFactory org.apache.lucene.analysis.fa.PersianNormalizationFilterFactory org.apache.lucene.analysis.fi.FinnishLightStemFilterFactory org.apache.lucene.analysis.fr.FrenchLightStemFilterFactory diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishMinimalStemFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishMinimalStemFilter.java new file mode 100644 index 00000000000..dd2619a3d10 --- /dev/null +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishMinimalStemFilter.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.analysis.es; + +import java.io.IOException; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.core.KeywordTokenizer; +import org.apache.lucene.analysis.en.EnglishMinimalStemFilter; + +/** + * Simple tests for {@link SpanishMinimalStemFilter} + */ +public class TestSpanishMinimalStemFilter extends BaseTokenStreamTestCase { + private Analyzer analyzer; + + @Override + public void setUp() throws Exception { + super.setUp(); + analyzer = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(source, new SpanishMinimalStemFilter(source)); + } + }; + } + + @Override + public void tearDown() throws Exception { + analyzer.close(); + super.tearDown(); + } + + /** Test some examples */ + public void testExamples() throws IOException { + checkOneTerm(analyzer, "actrices", "actriz"); + checkOneTerm(analyzer, "niños", "nino"); + checkOneTerm(analyzer, "países", "pais"); + checkOneTerm(analyzer, "caragodor", "caragodor"); + checkOneTerm(analyzer, "móviles", "movil"); + checkOneTerm(analyzer, "chicas", "chica"); + } + + /** blast some random strings through the analyzer */ + public void testRandomStrings() throws Exception { + checkRandomData(random(), analyzer, 50*RANDOM_MULTIPLIER); + } + + public void testEmptyTerm() throws IOException { + Analyzer a = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new KeywordTokenizer(); + return new TokenStreamComponents(tokenizer, new EnglishMinimalStemFilter(tokenizer)); + } + }; + checkOneTerm(a, "", ""); + a.close(); + } + +} diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishMinimalStemFilterFactory.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishMinimalStemFilterFactory.java new file mode 100644 index 00000000000..aed87dccfa3 --- /dev/null +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishMinimalStemFilterFactory.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.analysis.es; + +import java.io.Reader; +import java.io.StringReader; + +import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase; + +/** + * Simple tests to ensure the spanish minimal stem factory is working. + */ +public class TestSpanishMinimalStemFilterFactory extends BaseTokenStreamFactoryTestCase { + public void testStemming() throws Exception { + Reader reader = new StringReader("camisetas"); + TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false); + ((Tokenizer)stream).setReader(reader); + stream = tokenFilterFactory("spanishMinimalStem").create(stream); + assertTokenStreamContents(stream, new String[] { "camiseta" }); + } + + /** Test that bogus arguments result in exception */ + public void testBogusArguments() throws Exception { + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> { + tokenFilterFactory("spanishMinimalStem", "bogusArg", "bogusValue"); + }); + assertTrue(expected.getMessage().contains("Unknown parameters")); + } +} From cab83772c9e2f6e654a21163025ec87a3635dca6 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Fri, 26 Jul 2019 05:50:29 -0400 Subject: [PATCH 06/38] SOLR-13629: Cleanup whitespace in analytics contrib --- solr/CHANGES.txt | 2 + .../solr/analytics/AnalyticsDriver.java | 10 +- .../solr/analytics/AnalyticsExpression.java | 14 +- .../analytics/AnalyticsGroupingManager.java | 48 +-- .../analytics/AnalyticsRequestManager.java | 82 ++-- .../analytics/AnalyticsRequestParser.java | 106 ++--- .../solr/analytics/ExpressionFactory.java | 116 ++--- .../facet/AbstractSolrQueryFacet.java | 20 +- .../solr/analytics/facet/AnalyticsFacet.java | 40 +- .../solr/analytics/facet/PivotFacet.java | 18 +- .../solr/analytics/facet/PivotNode.java | 32 +- .../solr/analytics/facet/QueryFacet.java | 4 +- .../solr/analytics/facet/RangeFacet.java | 10 +- .../solr/analytics/facet/SortableFacet.java | 36 +- .../solr/analytics/facet/ValueFacet.java | 6 +- .../facet/compare/DelegatingComparator.java | 6 +- .../facet/compare/ExpressionComparator.java | 6 +- .../facet/compare/FacetResultsComparator.java | 10 +- .../facet/compare/FacetValueComparator.java | 4 +- .../analytics/facet/compare/package-info.java | 4 +- .../solr/analytics/facet/package-info.java | 4 +- .../function/ExpressionCalculator.java | 8 +- .../MergingReductionCollectionManager.java | 8 +- .../function/ReductionCollectionManager.java | 90 ++-- .../analytics/function/ReductionFunction.java | 2 +- .../function/field/AnalyticsField.java | 22 +- .../function/field/BooleanField.java | 4 +- .../function/field/BooleanMultiField.java | 8 +- .../analytics/function/field/DateField.java | 4 +- .../function/field/DateMultiPointField.java | 2 +- .../function/field/DateMultiTrieField.java | 2 +- .../analytics/function/field/DoubleField.java | 4 +- .../function/field/DoubleMultiPointField.java | 6 +- .../function/field/DoubleMultiTrieField.java | 6 +- .../analytics/function/field/FloatField.java | 4 +- .../function/field/FloatMultiPointField.java | 8 +- .../function/field/FloatMultiTrieField.java | 8 +- .../analytics/function/field/IntField.java | 6 +- .../function/field/IntMultiPointField.java | 8 +- .../function/field/IntMultiTrieField.java | 8 +- .../function/field/LongMultiPointField.java | 8 +- .../function/field/LongMultiTrieField.java | 6 +- .../function/field/StringMultiField.java | 4 +- .../function/field/package-info.java | 4 +- .../function/mapping/AddFunction.java | 6 +- .../function/mapping/BottomFunction.java | 2 +- .../function/mapping/ComparisonFunction.java | 18 +- .../function/mapping/ConcatFunction.java | 8 +- .../function/mapping/DateMathFunction.java | 6 +- .../function/mapping/DateParseFunction.java | 10 +- .../DecimalNumericConversionFunction.java | 22 +- .../function/mapping/DivideFunction.java | 4 +- .../function/mapping/EqualFunction.java | 14 +- .../function/mapping/ExistsFunction.java | 10 +- .../function/mapping/FillMissingFunction.java | 78 ++-- .../function/mapping/FilterFunction.java | 82 ++-- .../function/mapping/IfFunction.java | 42 +- .../function/mapping/LambdaFunction.java | 408 +++++++++--------- .../function/mapping/LogFunction.java | 4 +- .../function/mapping/LogicFunction.java | 10 +- .../function/mapping/MultFunction.java | 6 +- .../function/mapping/PowerFunction.java | 4 +- .../function/mapping/RemoveFunction.java | 94 ++-- .../function/mapping/ReplaceFunction.java | 84 ++-- .../function/mapping/SubtractFunction.java | 2 +- .../function/mapping/TopFunction.java | 2 +- .../function/mapping/package-info.java | 4 +- .../solr/analytics/function/package-info.java | 4 +- .../function/reduction/CountFunction.java | 4 +- .../function/reduction/DocCountFunction.java | 4 +- .../function/reduction/MaxFunction.java | 14 +- .../function/reduction/MeanFunction.java | 4 +- .../function/reduction/MedianFunction.java | 6 +- .../function/reduction/MinFunction.java | 14 +- .../function/reduction/MissingFunction.java | 2 +- .../function/reduction/OrdinalFunction.java | 16 +- .../reduction/PercentileFunction.java | 16 +- .../function/reduction/SumFunction.java | 4 +- .../function/reduction/UniqueFunction.java | 2 +- .../reduction/data/CountCollector.java | 28 +- .../function/reduction/data/MaxCollector.java | 62 +-- .../function/reduction/data/MinCollector.java | 62 +-- .../data/ReductionDataCollector.java | 70 +-- .../reduction/data/SortedListCollector.java | 76 ++-- .../function/reduction/data/SumCollector.java | 16 +- .../reduction/data/UniqueCollector.java | 42 +- .../function/reduction/data/package-info.java | 4 +- .../function/reduction/package-info.java | 4 +- .../apache/solr/analytics/package-info.java | 6 +- .../solr/analytics/plugin/package-info.java | 6 +- .../stream/AnalyticsShardRequestManager.java | 28 +- .../stream/AnalyticsShardResponseParser.java | 6 +- .../solr/analytics/stream/package-info.java | 4 +- .../reservation/BooleanArrayReservation.java | 2 +- .../BooleanCheckedReservation.java | 2 +- .../reservation/BooleanReservation.java | 2 +- .../reservation/DoubleArrayReservation.java | 2 +- .../reservation/DoubleCheckedReservation.java | 2 +- .../stream/reservation/DoubleReservation.java | 2 +- .../reservation/FloatArrayReservation.java | 2 +- .../reservation/FloatCheckedReservation.java | 2 +- .../stream/reservation/FloatReservation.java | 2 +- .../reservation/IntArrayReservation.java | 2 +- .../reservation/IntCheckedReservation.java | 2 +- .../stream/reservation/IntReservation.java | 2 +- .../reservation/LongArrayReservation.java | 2 +- .../reservation/LongCheckedReservation.java | 2 +- .../stream/reservation/LongReservation.java | 2 +- .../ReductionCheckedDataReservation.java | 2 +- .../ReductionDataArrayReservation.java | 2 +- .../reservation/ReductionDataReservation.java | 10 +- .../reservation/StringArrayReservation.java | 2 +- .../reservation/StringCheckedReservation.java | 2 +- .../stream/reservation/StringReservation.java | 2 +- .../stream/reservation/package-info.java | 4 +- .../read/BooleanCheckedDataReader.java | 2 +- .../read/BooleanDataArrayReader.java | 2 +- .../reservation/read/BooleanDataReader.java | 2 +- .../read/DoubleCheckedDataReader.java | 2 +- .../read/DoubleDataArrayReader.java | 2 +- .../reservation/read/DoubleDataReader.java | 2 +- .../read/FloatCheckedDataReader.java | 2 +- .../read/FloatDataArrayReader.java | 2 +- .../reservation/read/FloatDataReader.java | 2 +- .../read/IntCheckedDataReader.java | 2 +- .../reservation/read/IntDataArrayReader.java | 2 +- .../reservation/read/IntDataReader.java | 2 +- .../read/LongCheckedDataReader.java | 2 +- .../reservation/read/LongDataArrayReader.java | 2 +- .../reservation/read/LongDataReader.java | 2 +- .../read/ReductionCheckedDataReader.java | 10 +- .../read/ReductionDataArrayReader.java | 12 +- .../reservation/read/ReductionDataReader.java | 6 +- .../read/StringCheckedDataReader.java | 2 +- .../read/StringDataArrayReader.java | 2 +- .../reservation/read/StringDataReader.java | 2 +- .../stream/reservation/read/package-info.java | 4 +- .../write/BooleanCheckedDataWriter.java | 2 +- .../write/BooleanDataArrayWriter.java | 2 +- .../reservation/write/BooleanDataWriter.java | 2 +- .../write/DoubleCheckedDataWriter.java | 2 +- .../write/DoubleDataArrayWriter.java | 2 +- .../reservation/write/DoubleDataWriter.java | 2 +- .../write/FloatCheckedDataWriter.java | 2 +- .../write/FloatDataArrayWriter.java | 2 +- .../reservation/write/FloatDataWriter.java | 2 +- .../write/IntCheckedDataWriter.java | 2 +- .../reservation/write/IntDataArrayWriter.java | 2 +- .../reservation/write/IntDataWriter.java | 2 +- .../write/LongCheckedDataWriter.java | 2 +- .../write/LongDataArrayWriter.java | 2 +- .../reservation/write/LongDataWriter.java | 2 +- .../write/ReductionCheckedDataWriter.java | 12 +- .../write/ReductionDataArrayWriter.java | 12 +- .../write/ReductionDataWriter.java | 4 +- .../write/StringCheckedDataWriter.java | 2 +- .../write/StringDataArrayWriter.java | 2 +- .../reservation/write/StringDataWriter.java | 2 +- .../reservation/write/package-info.java | 4 +- .../util/AnalyticsResponseHeadings.java | 2 +- .../analytics/util/FacetRangeGenerator.java | 94 ++-- .../analytics/util/OldAnalyticsParams.java | 24 +- .../util/OldAnalyticsRequestConverter.java | 36 +- .../analytics/util/OrdinalCalculator.java | 2 +- .../analytics/util/function/package-info.java | 4 +- .../solr/analytics/util/package-info.java | 4 +- .../solr/analytics/value/AnalyticsValue.java | 8 +- .../analytics/value/AnalyticsValueStream.java | 36 +- .../solr/analytics/value/BooleanValue.java | 8 +- .../analytics/value/BooleanValueStream.java | 8 +- .../solr/analytics/value/ComparableValue.java | 2 +- .../solr/analytics/value/DateValue.java | 6 +- .../solr/analytics/value/DateValueStream.java | 6 +- .../solr/analytics/value/DoubleValue.java | 6 +- .../analytics/value/DoubleValueStream.java | 4 +- .../solr/analytics/value/FloatValue.java | 4 +- .../analytics/value/FloatValueStream.java | 4 +- .../apache/solr/analytics/value/IntValue.java | 4 +- .../solr/analytics/value/IntValueStream.java | 6 +- .../solr/analytics/value/LongValue.java | 4 +- .../solr/analytics/value/LongValueStream.java | 6 +- .../solr/analytics/value/StringValue.java | 4 +- .../analytics/value/StringValueStream.java | 4 +- .../value/constant/ConstantValue.java | 18 +- .../value/constant/package-info.java | 4 +- .../solr/analytics/value/package-info.java | 4 +- .../apache/solr/handler/AnalyticsHandler.java | 12 +- .../handler/component/AnalyticsComponent.java | 36 +- .../AnalyticsShardResponseWriter.java | 10 +- .../solr/analytics/ExpressionFactoryTest.java | 58 +-- .../apache/solr/analytics/NoFacetTest.java | 62 +-- .../solr/analytics/OverallAnalyticsTest.java | 24 +- .../solr/analytics/SolrAnalyticsTestCase.java | 62 +-- .../solr/analytics/facet/PivotFacetTest.java | 22 +- .../solr/analytics/facet/QueryFacetTest.java | 24 +- .../solr/analytics/facet/RangeFacetTest.java | 92 ++-- .../facet/SolrAnalyticsFacetTestCase.java | 14 +- .../solr/analytics/facet/ValueFacetTest.java | 90 ++-- .../field/AbstractAnalyticsFieldTest.java | 100 ++--- .../function/field/BooleanFieldsTest.java | 10 +- .../function/field/DateFieldsTest.java | 18 +- .../function/field/DoubleFieldsTest.java | 18 +- .../function/field/FloatFieldsTest.java | 18 +- .../function/field/IntFieldsTest.java | 18 +- .../function/field/LongFieldsTest.java | 18 +- .../function/field/StringFieldsTest.java | 10 +- .../mapping/AbsoluteValueFunctionTest.java | 24 +- .../function/mapping/AddFunctionTest.java | 28 +- .../function/mapping/AndFunctionTest.java | 32 +- .../function/mapping/BottomFunctionTest.java | 86 ++-- .../function/mapping/CeilingFunctionTest.java | 18 +- .../function/mapping/ConcatFunctionTest.java | 48 +-- .../mapping/DateMathFunctionTest.java | 14 +- .../mapping/DateParseFunctionTest.java | 28 +- .../function/mapping/DivideFunctionTest.java | 20 +- .../function/mapping/EqualFunctionTest.java | 56 +-- .../mapping/FillMissingFunctionTest.java | 56 +-- .../function/mapping/FilterFunctionTest.java | 132 +++--- .../function/mapping/FloorFunctionTest.java | 18 +- .../function/mapping/GTEFunctionTest.java | 30 +- .../function/mapping/GTFunctionTest.java | 30 +- .../function/mapping/IfFunctionTest.java | 112 ++--- .../function/mapping/LTEFunctionTest.java | 30 +- .../function/mapping/LTFunctionTest.java | 30 +- .../function/mapping/LogFunctionTest.java | 28 +- .../function/mapping/MultFunctionTest.java | 26 +- .../function/mapping/NegateFunctionTest.java | 30 +- .../function/mapping/OrFunctionTest.java | 32 +- .../function/mapping/PowerFunctionTest.java | 20 +- .../function/mapping/RemoveFunctionTest.java | 68 +-- .../function/mapping/ReplaceFunctionTest.java | 130 +++--- .../function/mapping/RoundFunctionTest.java | 18 +- .../mapping/StringCastFunctionTest.java | 6 +- .../mapping/SubtractFunctionTest.java | 20 +- .../function/mapping/TopFunctionTest.java | 86 ++-- .../LegacyAbstractAnalyticsCloudTest.java | 8 +- .../legacy/LegacyAbstractAnalyticsTest.java | 12 +- .../legacy/LegacyNoFacetCloudTest.java | 128 +++--- .../analytics/legacy/LegacyNoFacetTest.java | 118 ++--- .../legacy/expression/LegacyFunctionTest.java | 80 ++-- ...LegacyAbstractAnalyticsFacetCloudTest.java | 14 +- .../LegacyAbstractAnalyticsFacetTest.java | 24 +- .../legacy/facet/LegacyFacetSortingTest.java | 6 +- .../facet/LegacyFieldFacetCloudTest.java | 216 +++++----- .../LegacyFieldFacetExtrasCloudTest.java | 52 +-- .../facet/LegacyFieldFacetExtrasTest.java | 46 +- .../legacy/facet/LegacyFieldFacetTest.java | 190 ++++---- .../facet/LegacyQueryFacetCloudTest.java | 24 +- .../legacy/facet/LegacyQueryFacetTest.java | 18 +- .../facet/LegacyRangeFacetCloudTest.java | 62 +-- .../legacy/facet/LegacyRangeFacetTest.java | 62 +-- .../value/CastingAnalyticsValueTest.java | 6 +- .../value/CastingBooleanValueStreamTest.java | 10 +- .../value/CastingBooleanValueTest.java | 18 +- .../value/CastingDateValueStreamTest.java | 14 +- .../analytics/value/CastingDateValueTest.java | 22 +- .../value/CastingDoubleValueStreamTest.java | 10 +- .../value/CastingDoubleValueTest.java | 18 +- .../value/CastingFloatValueStreamTest.java | 14 +- .../value/CastingFloatValueTest.java | 24 +- .../value/CastingIntValueStreamTest.java | 22 +- .../analytics/value/CastingIntValueTest.java | 36 +- .../value/CastingLongValueStreamTest.java | 14 +- .../analytics/value/CastingLongValueTest.java | 24 +- .../value/CastingStringValueStreamTest.java | 6 +- .../value/CastingStringValueTest.java | 12 +- .../analytics/value/ConstantValueTest.java | 40 +- .../analytics/value/FillableTestValue.java | 94 ++-- 268 files changed, 3076 insertions(+), 3074 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 340eb7a2afe..c51fd8082ed 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -116,6 +116,8 @@ Other Changes * SOLR-10377: Add `debug.explain.structured` to Admin UI. (David Smiley, Munendra S N) +* SOLR-13629: Clean trailing whitespace from 'analytics' contrib (Neal Sidhwaney via Jason Gerlowski) + ================== 8.2.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsDriver.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsDriver.java index 21b053fdeef..21b7d1314e8 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsDriver.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsDriver.java @@ -31,10 +31,10 @@ import org.apache.solr.search.Filter; import org.apache.solr.search.SolrIndexSearcher; public class AnalyticsDriver { - + /** * Drive the collection of reduction data. This includes overall data as well as faceted data. - * + * * @param manager of the request to drive * @param searcher the results of the query * @param filter that represents the overall query @@ -45,9 +45,9 @@ public class AnalyticsDriver { StreamingInfo streamingInfo = manager.getStreamingFacetInfo(); Iterable streamingFacets = streamingInfo.streamingFacets; ReductionCollectionManager collectionManager = streamingInfo.streamingCollectionManager; - + Iterable facetExecuters = manager.getFacetExecuters(filter, queryRequest); - + // Streaming phase (Overall results & Value/Pivot Facets) // Loop through all documents and collect reduction data for streaming facets and overall results if (collectionManager.needsCollection()) { @@ -72,7 +72,7 @@ public class AnalyticsDriver { } } } - + // Executing phase (Query/Range Facets) // Send additional Solr Queries to compute facet values for (FacetValueQueryExecuter executer : facetExecuters) { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsExpression.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsExpression.java index 044e371d063..fcf393e9c49 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsExpression.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsExpression.java @@ -27,35 +27,35 @@ import org.apache.solr.analytics.value.AnalyticsValue; public class AnalyticsExpression { private final AnalyticsValue expression; private final String name; - + public AnalyticsExpression(String name, AnalyticsValue expression) { this.name = name; this.expression = expression; } - + public String getName() { return name; } - + public AnalyticsValue getExpression() { return expression; } - + /** * Get the current value of the expression. * This method can, and will, be called multiple times to return different values. * The value returned is based on the {@link ReductionDataCollection} given * to the {@link ReductionCollectionManager#setData} method. - * + * * @return the current value of the expression */ public Object toObject() { return expression.getObject(); } - + /** * NOTE: Must be called after {@link #toObject()} is called, otherwise the value is not guaranteed to be correct. - * + * * @return whether the current value of the expression exists. */ public boolean exists() { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsGroupingManager.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsGroupingManager.java index a95a4516349..75b3ad9743e 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsGroupingManager.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsGroupingManager.java @@ -43,7 +43,7 @@ import org.apache.solr.search.Filter; /** * The manager for faceted analytics. This class manages one grouping of facets and expressions to compute * over those facets. - * + * *

* This class will only manage generating faceted results, not overall results. */ @@ -53,9 +53,9 @@ public class AnalyticsGroupingManager { private final Collection topLevelExpressions; private final ExpressionCalculator expressionCalculator; - + private final Map facets; - + public AnalyticsGroupingManager(String name, ReductionCollectionManager reductionCollectionManager, Collection topLevelExpressions) { @@ -70,11 +70,11 @@ public class AnalyticsGroupingManager { // This is outside of the method, since it is used in the lambda and cannot be a local non-final variable private boolean hasStreamingFacets; - + /** * Get the {@link StreamingFacet}s (e.g. {@link ValueFacet} and {@link PivotFacet}) contained within this grouping, * returning them through the given consumer. - * + * * @param cons where the streaming facets are passed to * @return whether the grouping contains streaming facets */ @@ -93,12 +93,12 @@ public class AnalyticsGroupingManager { * Create the {@link FacetValueQueryExecuter}s for all {@link AbstractSolrQueryFacet}s * (e.g. {@link QueryFacet} and {@link RangeFacet}) contained within this grouping. * The executers are returned through the given consumer. - * + * *

* One {@link FacetValueQueryExecuter} is created for each facet value to be returned for a facet. * Since every {@link AbstractSolrQueryFacet} has discrete and user-defined facet values, * unlike {@link StreamingFacet}s, a discrete number of {@link FacetValueQueryExecuter}s are created and returned. - * + * * @param filter representing the overall Solr Query of the request, * will be combined with the facet value queries * @param queryRequest from the overall search request @@ -111,10 +111,10 @@ public class AnalyticsGroupingManager { } }); } - + /** * Add a facet to the grouping. All expressions in this grouping will be computed over the facet. - * + * * @param facet to compute expressions over */ public void addFacet(AnalyticsFacet facet) { @@ -122,11 +122,11 @@ public class AnalyticsGroupingManager { facet.setReductionCollectionManager(reductionCollectionManager); facets.put(facet.getName(), facet); } - + /** * Import the shard data for this grouping from a bit-stream, * exported by the {@link #exportShardData} method in the each of the collection's shards. - * + * * @param input The bit-stream to import the grouping data from * @throws IOException if an exception occurs while reading from the {@link DataInput} */ @@ -134,17 +134,17 @@ public class AnalyticsGroupingManager { // This allows mergeData() to import from the same input everytime it is called // while the facets are importing. reductionCollectionManager.setShardInput(input); - + int sz = input.readInt(); for (int i = 0; i < sz; ++i) { facets.get(input.readUTF()).importShardData(input); } } - + /** * Export the shard data for this grouping through a bit-stream, * to be imported by the {@link #importShardData} method in the originating shard. - * + * * @param output The bit-stream to output the grouping data through * @throws IOException if an exception occurs while writing to the {@link DataOutput} */ @@ -152,18 +152,18 @@ public class AnalyticsGroupingManager { // This allows exportData() to export to the same output everytime it is called // while the facets are exporting. reductionCollectionManager.setShardOutput(output); - + output.writeInt(facets.size()); for (Entry facet : facets.entrySet()) { output.writeUTF(facet.getKey()); facet.getValue().exportShardData(output); } } - + /** * Get the {@link ReductionCollectionManager} that manages the collection of reduction data for the expressions - * contained within this grouping. - * + * contained within this grouping. + * * @return the grouping's reduction manager */ public ReductionCollectionManager getReductionManager() { @@ -172,18 +172,18 @@ public class AnalyticsGroupingManager { /** * Create the response for this grouping, a mapping from each of it's facets' names to the facet's response. - * + * * @return the named list representation of the response */ public Map createResponse() { Map response = new HashMap<>(); - + // Add the value facet buckets to the output facets.forEach( (name, facet) -> response.put(name, facet.createResponse()) ); return response; } - + /** * Create the response for this grouping, but in the old style of response. * This response has a bucket for the following if they are contained in the grouping: @@ -192,13 +192,13 @@ public class AnalyticsGroupingManager { *

* Since groupings in the old notation must also return overall results, the overall results are * passed in and the values are used to populate the grouping response. - * + * * @param overallResults of the expressions to add to the grouping response * @return the named list representation of the response */ public NamedList createOldResponse(Map overallResults) { NamedList response = new NamedList<>(); - + topLevelExpressions.forEach( expression -> response.add(expression.getName(), overallResults.get(name + expression.getName()))); NamedList fieldFacetResults = new NamedList<>(); @@ -230,7 +230,7 @@ public class AnalyticsGroupingManager { /** * Get the name of the grouping. - * + * * @return the grouping name */ public String getName() { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsRequestManager.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsRequestManager.java index 45b958f14d0..f1bfba82068 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsRequestManager.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsRequestManager.java @@ -42,25 +42,25 @@ import org.apache.solr.search.Filter; public class AnalyticsRequestManager { private final ReductionCollectionManager ungroupedReductionManager; private ReductionDataCollection ungroupedData; - + private final Map groupingManagers; - + private final Collection ungroupedExpressions; private final ExpressionCalculator ungroupedExpressionCalculator; - + /** * If the request is distributed, the manager for shard requests. */ public String analyticsRequest; public AnalyticsShardRequestManager shardStream; - public boolean sendShards; - + public boolean sendShards; + /** * Create an manager with the given ungrouped expressions. This is straightforward in the new * style of request, however in the old olap-style requests all groupings' expressions are expected * to be ungrouped as well. - * - * + * + * * @param ungroupedReductionManager to manage the reduction collection for all ungrouped expressions * @param ungroupedExpressions to compute overall results for */ @@ -69,85 +69,85 @@ public class AnalyticsRequestManager { this.ungroupedReductionManager = ungroupedReductionManager; this.ungroupedData = ungroupedReductionManager.newDataCollection(); this.ungroupedReductionManager.addLastingCollectTarget(ungroupedData); - + this.ungroupedExpressions = ungroupedExpressions; this.ungroupedExpressionCalculator = new ExpressionCalculator(ungroupedExpressions); this.groupingManagers = new HashMap<>(); } - + /** * Get the collection manager for ungrouped expressions, including grouped expressions if * the old request notation is used. - * + * * @return the collection manager for the ungrouped expressions */ public ReductionCollectionManager getUngroupedCollectionManager() { return ungroupedReductionManager; } - + /** * Get the collection manager for all ungrouped expressions, including grouped expressions if * the old request notation is used. - * + * * @return the collection manager for the ungrouped expressions */ public ReductionDataCollection getUngroupedData() { return ungroupedData; } - + /** * Return all ungrouped expressions, including grouped expressions if * the old request notation is used. - * + * * @return an {@link Iterable} of the ungrouped expressions */ public Iterable getUngroupedExpressions() { return ungroupedExpressions; } - + /** * Generate the results of all ungrouped expressions, including grouped expressions if * the old request notation is used. - * + * * @param response the response to add the ungrouped results to. */ public void addUngroupedResults(Map response) { ungroupedReductionManager.setData(ungroupedData); ungroupedExpressionCalculator.addResults(response); } - + /** * Generate the results of all ungrouped expressions, including grouped expressions if * the old request notation is used. - * + * * @return the map containing the ungrouped results */ public Map getUngroupedResults() { ungroupedReductionManager.setData(ungroupedData); return ungroupedExpressionCalculator.getResults(); } - + /** * Add a grouping to the request. - * + * * @param groupingManager that manages the grouping */ public void addGrouping(AnalyticsGroupingManager groupingManager) { groupingManagers.put(groupingManager.getName(), groupingManager); } - + /** * Import the shard data for this request from a bit-stream, * exported by the {@link #exportShardData} method in the each of the collection's shards. *

* First the overall data is imported, then the grouping data is imported. - * + * * @param input The bit-stream to import the shard data from * @throws IOException if an exception occurs while reading from the {@link DataInput} */ public synchronized void importShardData(DataInput input) throws IOException { ungroupedReductionManager.setShardInput(input); - + // The ungroupedData will not exist for the first shard imported if (ungroupedData == null) { ungroupedData = ungroupedReductionManager.newDataCollectionIO(); @@ -155,29 +155,29 @@ public class AnalyticsRequestManager { ungroupedReductionManager.prepareReductionDataIO(ungroupedData); } ungroupedReductionManager.mergeData(); - + int size = input.readInt(); while (--size >= 0) { String groupingName = input.readUTF(); groupingManagers.get(groupingName).importShardData(input); } } - + /** * Export the shard data for this request through a bit-stream, * to be imported by the {@link #importShardData} method in the originating shard. *

* First the overall data is exported, then the grouping data is exported. - * + * * @param output The bit-stream to output the shard data through * @throws IOException if an exception occurs while writing to the {@link DataOutput} */ public void exportShardData(DataOutput output) throws IOException { ungroupedReductionManager.setShardOutput(output); - + ungroupedReductionManager.prepareReductionDataIO(ungroupedData); ungroupedReductionManager.exportData(); - + output.writeInt(groupingManagers.size()); for (String groupingName : groupingManagers.keySet()) { output.writeUTF(groupingName); @@ -187,9 +187,9 @@ public class AnalyticsRequestManager { /** * Consolidate the information of all {@link StreamingFacet}s contained within the request, since - * they need to be collected along with the overall results during the streaming phase of the + * they need to be collected along with the overall results during the streaming phase of the * {@link AnalyticsDriver}. - * + * * @return the info for all {@link StreamingFacet}s */ public StreamingInfo getStreamingFacetInfo() { @@ -197,18 +197,18 @@ public class AnalyticsRequestManager { ArrayList groupingCollectors = new ArrayList<>(); groupingManagers.values().forEach( grouping -> { // If a grouping has streaming facets, then that groupings expressions - // must be collected during the streaming phase. + // must be collected during the streaming phase. if (grouping.getStreamingFacets( facet -> streamingInfo.streamingFacets.add(facet) )) { groupingCollectors.add(grouping.getReductionManager()); } }); - + // Create an streaming collection manager to manage the collection of all ungrouped expressions and // grouped expressions that are calculated over streaming facets. streamingInfo.streamingCollectionManager = ungroupedReductionManager.merge(groupingCollectors); return streamingInfo; } - + /** * Class to encapsulate all necessary data for collecting {@link StreamingFacet}s. */ @@ -222,7 +222,7 @@ public class AnalyticsRequestManager { /** * Create the {@link FacetValueQueryExecuter}s for all {@link AbstractSolrQueryFacet}s contained in the request. - * + * * @param filter representing the overall search query * @param queryRequest of the overall search query * @return an {@link Iterable} of executers @@ -234,11 +234,11 @@ public class AnalyticsRequestManager { }); return facetExecutors; } - + /** * Create the response for a request given in the old olap-style format. * The old response returned overall expressions within groupings. - * + * * @return a {@link NamedList} representation of the response */ public NamedList createOldResponse() { @@ -247,17 +247,17 @@ public class AnalyticsRequestManager { groupingManagers.forEach( (name, groupingManager) -> { analyticsResponse.add(name, groupingManager.createOldResponse(ungroupedResults)); }); - + return analyticsResponse; } - + /** * Create the response for a request. - * + * *

* NOTE: Analytics requests specified in the old olap-style format * have their responses generated by {@link #createOldResponse()}. - * + * * @return a {@link Map} representation of the response */ public Map createResponse() { @@ -270,7 +270,7 @@ public class AnalyticsRequestManager { groupingManagers.forEach( (name, groupingManager) -> { groupingsResponse.put(name, groupingManager.createResponse()); }); - + if (groupingsResponse.size() > 0) { analyticsResponse.put(AnalyticsResponseHeadings.GROUPINGS, groupingsResponse); } diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsRequestParser.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsRequestParser.java index bcb77473cf6..546cabab3f7 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsRequestParser.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/AnalyticsRequestParser.java @@ -63,43 +63,43 @@ import com.fasterxml.jackson.databind.ObjectMapper; * Class to manage the parsing of new-style analytics requests. */ public class AnalyticsRequestParser { - + private static ObjectMapper mapper = new ObjectMapper(); - + public static void init() { mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); mapper.configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, true); } - + public static final String analyticsParamName = "analytics"; private static Predicate sortAscending = acceptNames("ascending", "asc", "a"); private static Predicate sortDescending = acceptNames("descending", "desc", "d"); - + private static Predicate acceptNames(String... names) { return Pattern.compile("^(?:" + Arrays.stream(names).reduce((a,b) -> a + "|" + b).orElse("") + ")$", Pattern.CASE_INSENSITIVE).asPredicate(); } - + // Defaults public static final String DEFAULT_SORT_DIRECTION = "ascending"; public static final int DEFAULT_OFFSET = 0; public static final int DEFAULT_LIMIT = -1; public static final boolean DEFAULT_HARDEND = false; - + @JsonInclude(Include.NON_EMPTY) public static class AnalyticsRequest { public Map functions; public Map expressions; - + public Map groupings; } - + public static class AnalyticsGroupingRequest { public Map expressions; - + public Map facets; } - + @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, @@ -113,7 +113,7 @@ public class AnalyticsRequestParser { ) @JsonInclude(Include.NON_EMPTY) public static interface AnalyticsFacetRequest { } - + @JsonTypeName("value") public static class AnalyticsValueFacetRequest implements AnalyticsFacetRequest { public String expression; @@ -124,7 +124,7 @@ public class AnalyticsRequestParser { public static class AnalyticsPivotFacetRequest implements AnalyticsFacetRequest { public List pivots; } - + public static class AnalyticsPivotRequest { public String name; public String expression; @@ -175,11 +175,11 @@ public class AnalyticsRequestParser { public static class AnalyticsQueryFacetRequest implements AnalyticsFacetRequest { public Map queries; } - + /* *************** - * Request & Groupings + * Request & Groupings * ***************/ - + public static AnalyticsRequestManager parse(AnalyticsRequest request, ExpressionFactory expressionFactory, boolean isDistribRequest) throws SolrException { AnalyticsRequestManager manager = constructRequest(request, expressionFactory, isDistribRequest); if (isDistribRequest) { @@ -191,7 +191,7 @@ public class AnalyticsRequestParser { } return manager; } - + public static AnalyticsRequestManager parse(String rawRequest, ExpressionFactory expressionFactory, boolean isDistribRequest) throws SolrException { JsonParser parser; try { @@ -214,15 +214,15 @@ public class AnalyticsRequestParser { } return manager; } - + private static AnalyticsRequestManager constructRequest(AnalyticsRequest request, ExpressionFactory expressionFactory, boolean isDistribRequest) throws SolrException { expressionFactory.startRequest(); - + // Functions if (request.functions != null) { request.functions.forEach( (funcSig, retSig) -> expressionFactory.addUserDefinedVariableFunction(funcSig, retSig)); } - + // Expressions Map topLevelExpressions; if (request.expressions != null) { @@ -231,7 +231,7 @@ public class AnalyticsRequestParser { topLevelExpressions = new HashMap<>(); } AnalyticsRequestManager manager = new AnalyticsRequestManager(expressionFactory.createReductionManager(isDistribRequest), topLevelExpressions.values()); - + // Groupings if (request.groupings != null) { request.groupings.forEach( (name, grouping) -> { @@ -240,20 +240,20 @@ public class AnalyticsRequestParser { } return manager; } - + private static AnalyticsGroupingManager constructGrouping(String name, AnalyticsGroupingRequest grouping, ExpressionFactory expressionFactory, boolean isDistribRequest) throws SolrException { expressionFactory.startGrouping(); - + // Expressions if (grouping.expressions == null || grouping.expressions.size() == 0) { throw new SolrException(ErrorCode.BAD_REQUEST,"Groupings must contain at least one expression, '" + name + "' has none."); } - + Map expressions = constructExpressions(grouping.expressions, expressionFactory); - AnalyticsGroupingManager manager = new AnalyticsGroupingManager(name, - expressionFactory.createGroupingReductionManager(isDistribRequest), + AnalyticsGroupingManager manager = new AnalyticsGroupingManager(name, + expressionFactory.createGroupingReductionManager(isDistribRequest), expressions.values()); - + if (grouping.facets == null) { throw new SolrException(ErrorCode.BAD_REQUEST,"Groupings must contain at least one facet, '" + name + "' has none."); } @@ -269,17 +269,17 @@ public class AnalyticsRequestParser { manager.addFacet(constructQueryFacet(facetName, (AnalyticsQueryFacetRequest) facet)); } else { throw new SolrException(ErrorCode.BAD_REQUEST,"The facet type, '" + facet.getClass().toString() + "' in " - + "grouping '" + name + "' is not a valid type of facet"); + + "grouping '" + name + "' is not a valid type of facet"); } }); - + return manager; } - + /* *************** - * Expression & Functions + * Expression & Functions * ***************/ - + private static Map constructExpressions(Map rawExpressions, ExpressionFactory expressionFactory) throws SolrException { Map expressions = new HashMap<>(); rawExpressions.forEach( (name, expression) -> { @@ -296,20 +296,20 @@ public class AnalyticsRequestParser { }); return expressions; } - + /* *************** - * FACETS + * FACETS * ***************/ - + /* * Value Facets */ - + private static ValueFacet constructValueFacet(String name, AnalyticsValueFacetRequest facetRequest, ExpressionFactory expressionFactory, Map expressions) throws SolrException { if (facetRequest.expression == null) { throw new SolrException(ErrorCode.BAD_REQUEST, "Value Facets must contain a mapping expression to facet over, '" + name + "' has none."); } - + // The second parameter must be a mapping expression AnalyticsValueStream expr = expressionFactory.createExpression(facetRequest.expression); if (!expr.getExpressionType().isUnreduced()) { @@ -320,9 +320,9 @@ public class AnalyticsRequestParser { throw new SolrException(ErrorCode.BAD_REQUEST, "Value Facet expressions must be castable to string expressions, " + "the following expression in value facet '" + name + "' is not: " + facetRequest.expression); } - + ValueFacet facet = new ValueFacet(name, (StringValueStream)expr); - + // Check if the value facet is sorted if (facetRequest.sort != null) { facet.setSort(constructSort(facetRequest.sort, expressions)); @@ -333,23 +333,23 @@ public class AnalyticsRequestParser { /* * Pivot Facets */ - + private static PivotFacet constructPivotFacet(String name, AnalyticsPivotFacetRequest facetRequest, ExpressionFactory expressionFactory, Map expressions) throws SolrException { PivotNode topPivot = null; - + // Pivots if (facetRequest.pivots == null || facetRequest.pivots.size() == 0) { throw new SolrException(ErrorCode.BAD_REQUEST, "Pivot Facets must contain at least one pivot to facet over, '" + name + "' has none."); } - + ListIterator iter = facetRequest.pivots.listIterator(facetRequest.pivots.size()); while (iter.hasPrevious()) { topPivot = constructPivot(iter.previous(), topPivot, expressionFactory, expressions); } - + return new PivotFacet(name, topPivot); } - + @SuppressWarnings({"unchecked", "rawtypes"}) private static PivotNode constructPivot(AnalyticsPivotRequest pivotRequest, PivotNode childPivot, @@ -361,7 +361,7 @@ public class AnalyticsRequestParser { if (pivotRequest.expression == null) { throw new SolrException(ErrorCode.BAD_REQUEST, "Pivots must have an expression to facet over, '" + pivotRequest.name + "' does not."); } - + // The second parameter must be a mapping expression AnalyticsValueStream expr = expressionFactory.createExpression(pivotRequest.expression); if (!expr.getExpressionType().isUnreduced()) { @@ -372,21 +372,21 @@ public class AnalyticsRequestParser { throw new SolrException(ErrorCode.BAD_REQUEST, "Pivot expressions must be castable to string expressions, " + "the following expression in pivot '" + pivotRequest.name + "' is not: '" + pivotRequest.expression); } - + PivotNode pivot; if (childPivot == null) { pivot = new PivotNode.PivotLeaf(pivotRequest.name, (StringValueStream)expr); } else { pivot = new PivotNode.PivotBranch(pivotRequest.name, (StringValueStream)expr, childPivot); } - + // Check if the pivot is sorted if (pivotRequest.sort != null) { pivot.setSort(constructSort(pivotRequest.sort, expressions)); } return pivot; } - + /* * Range Facets */ @@ -454,20 +454,20 @@ public class AnalyticsRequestParser { /* * Query Facets */ - + private static QueryFacet constructQueryFacet(String name, AnalyticsQueryFacetRequest facetRequest) throws SolrException { if (facetRequest.queries == null || facetRequest.queries.size() == 0) { throw new SolrException(ErrorCode.BAD_REQUEST, "Query Facets must be contain at least 1 query to facet over, '" + name + "' does not."); } - + // The first param must be the facet name return new QueryFacet(name, facetRequest.queries); } - + /* * Facet Sorting */ - + private static FacetSortSpecification constructSort(AnalyticsSortRequest sortRequest, Map expressions) throws SolrException { if (sortRequest.criteria == null || sortRequest.criteria.size() == 0) { throw new SolrException(ErrorCode.BAD_REQUEST, "Sorts must be given at least 1 criteria."); @@ -501,7 +501,7 @@ public class AnalyticsRequestParser { } return DelegatingComparator.joinComparators(comparators); } - + private static FacetResultsComparator constructExpressionSortCriteria(AnalyticsExpressionSortRequest criterion, Map expressions) { if (criterion.expression == null || criterion.expression.length() == 0) { throw new SolrException(ErrorCode.BAD_REQUEST,"Expression Sorts must contain an expression parameter, none given."); @@ -516,7 +516,7 @@ public class AnalyticsRequestParser { } return ((ComparableValue)expression.getExpression()).getObjectComparator(expression.getName()); } - + private static FacetResultsComparator constructFacetValueSortCriteria(AnalyticsFacetValueSortRequest criterion) { return new FacetValueComparator(); } diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/ExpressionFactory.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/ExpressionFactory.java index 18020b79d97..9407d1dd354 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/ExpressionFactory.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/ExpressionFactory.java @@ -66,7 +66,7 @@ import org.apache.solr.schema.TrieLongField; /** * A factory to parse and create expressions, and capture information about those expressions along the way. - * + * *

* In order to use, first call {@link #startRequest()} and create all ungrouped expressions, * then call {@link #createReductionManager} to get the ungrouped reduction manager. @@ -95,40 +95,40 @@ public class ExpressionFactory { private HashMap systemVariableFunctions; private HashMap variableFunctions; private HashSet variableFunctionNameHistory; - + private HashMap expressionCreators; private final ConstantFunction constantCreator; - + private LinkedHashMap reductionFunctions; private LinkedHashMap> collectors; private LinkedHashMap fields; private HashMap expressions; - + private IndexSchema schema; - + private Map> groupedCollectors; private Map groupedFields; private boolean isGrouped; public ExpressionFactory(IndexSchema schema) { this.schema = schema; - + expressionCreators = new HashMap<>(); systemVariableFunctions = new HashMap<>(); - + constantCreator = ConstantValue.creatorFunction; addSystemFunctions(); } - + /** * Get the index schema used by this factory. - * + * * @return the index schema */ public IndexSchema getSchema() { return schema; } - + /** * Prepare the factory to start building the request. */ @@ -141,10 +141,10 @@ public class ExpressionFactory { variableFunctions = new HashMap<>(); variableFunctions.putAll(systemVariableFunctions); variableFunctionNameHistory = new HashSet<>(); - + isGrouped = false; } - + /** * Prepare the factory to start building the next grouping. *
@@ -153,14 +153,14 @@ public class ExpressionFactory { public void startGrouping() { groupedCollectors = new HashMap<>(); groupedFields = new HashMap<>(); - + isGrouped = true; } /** * Add a system function to the expression factory. * This will be treated as a native function and not a variable function. - * + * * @param functionName the unique name for the function * @param functionCreator the creator function to generate an expression * @return this factory, to easily chain function adds @@ -176,7 +176,7 @@ public class ExpressionFactory { /** * Add a variable function that will be treated like a system function. - * + * * @param functionName the function's name * @param functionParams the comma separated and ordered parameters of the function (e.g. {@code "a,b"} ) * @param returnSignature the return signature of the variable function (e.g. {@code div(sum(a,b),count(b))} ) @@ -184,7 +184,7 @@ public class ExpressionFactory { * @throws SolrException if the name of the function is not unique or the syntax of either signature is incorrect */ public ExpressionFactory addSystemVariableFunction(final String functionName, final String functionParams, final String returnSignature) throws SolrException { - return addVariableFunction(functionName, + return addVariableFunction(functionName, Arrays.stream(functionParams.split(",")).map(param -> param.trim()).toArray(size -> new String[size]), returnSignature, systemVariableFunctions); @@ -192,7 +192,7 @@ public class ExpressionFactory { /** * Add a variable function that was defined in an analytics request. - * + * * @param functionSignature the function signature of the variable function (e.g. {@code func(a,b)} ) * @param returnSignature the return signature of the variable function (e.g. {@code div(sum(a,b),count(b))} ) * @return this factory, to easily chain function adds @@ -201,10 +201,10 @@ public class ExpressionFactory { public ExpressionFactory addUserDefinedVariableFunction(final String functionSignature, final String returnSignature) throws SolrException { return addVariableFunction(functionSignature, returnSignature, variableFunctions); } - + /** - * Add a variable function to the given map of variable functions. - * + * Add a variable function to the given map of variable functions. + * * @param functionSignature the function signature of the variable function (e.g. {@code func(a,b)} ) * @param returnSignature the return signature of the variable function (e.g. {@code div(sum(a,b),count(b))} ) * @param variableFunctions the map of variable functions to add the new function to @@ -217,10 +217,10 @@ public class ExpressionFactory { addVariableFunction(getFunctionName(functionSignature), getParams(functionSignature, null, null), returnSignature, variableFunctions); return this; } - + /** - * Add a variable function to the given map of variable functions. - * + * Add a variable function to the given map of variable functions. + * * @param functionName the function's name * @param functionParams the parameters of the function (this is ordered) * @param returnSignature the return signature of the variable function (e.g. {@code div(sum(a,b),count(b))} ) @@ -245,7 +245,7 @@ public class ExpressionFactory { /** * Create a reduction manager to manage the collection of all expressions that have been created since * {@link #startRequest()} was called. - * + * * @param isCloudCollection whether the request is a distributed request * @return a reduction manager */ @@ -262,7 +262,7 @@ public class ExpressionFactory { /** * Create a reduction manager to manage the collection of all expressions that have been created since * {@link #startGrouping()} was called. - * + * * @param isCloudCollection whether the request is a distributed request * @return a reduction manager */ @@ -275,10 +275,10 @@ public class ExpressionFactory { return new ReductionCollectionManager(collectorsArr, groupedFields.values()); } } - + /** * Parse and build an expression from the given expression string. - * + * * @param expressionStr string that represents the desired expression * @return the object representation of the expression * @throws SolrException if an error occurs while constructing the expression @@ -289,7 +289,7 @@ public class ExpressionFactory { /** * Create an expression from the given expression string, with the given variable function information. - * + * * @param expressionStr string that represents the desired expression * @param varFuncParams the current set of variable function parameters and their values. If this expression is not a variable function * return signature, the map should be empty. @@ -304,7 +304,7 @@ public class ExpressionFactory { String varFuncVarParamName, String[] varFuncVarParamValues) throws SolrException { AnalyticsValueStream expression; expressionStr = expressionStr.trim(); - + boolean isField = false; try { // Try to make a constant value @@ -363,10 +363,10 @@ public class ExpressionFactory { } return expression; } - + /** * Create a function expression from the given expression string, with the given variable function information. - * + * * @param expressionStr string that represents the desired expression * @param varFuncParams the current set of variable function parameters and their values. If this expression is not a variable function * return signature, the map should be empty. @@ -384,9 +384,9 @@ public class ExpressionFactory { final String[] params = getParams(expressionStr, varFuncVarParamName, varFuncVarParamValues); AnalyticsValueStream[] paramStreams = new AnalyticsValueStream[params.length]; - + boolean allParamsConstant = true; - + for (int i = 0; i < params.length; i++) { // First check if the parameter is a variable function variable otherwise create the expression if (varFuncParams.containsKey(params[i])) { @@ -394,22 +394,22 @@ public class ExpressionFactory { } else { paramStreams[i] = createExpression(params[i], varFuncParams, varFuncVarParamName, varFuncVarParamValues); } - + // Then update whether all of the params are constant allParamsConstant &= paramStreams[i].getExpressionType().equals(ExpressionType.CONST); } - + // Check to see if the function name is a variable function name, if so apply the variables to the return signature if (variableFunctions.containsKey(name)) { if (variableFunctionNameHistory.contains(name)) { - throw new SolrException(ErrorCode.BAD_REQUEST,"The following variable function is self referencing : " + name); + throw new SolrException(ErrorCode.BAD_REQUEST,"The following variable function is self referencing : " + name); } variableFunctionNameHistory.add(name); VariableFunctionInfo newVarFunc = variableFunctions.get(name); Map newVarFuncParams = new HashMap<>(); - + boolean varLenEnd = false; - + if (paramStreams.length < newVarFunc.params.length) { throw new SolrException(ErrorCode.BAD_REQUEST,"The variable function '" + name + "' requires at least " + newVarFunc.params.length + " parameters." + " Only " + paramStreams.length + " arguments given in the following invocation : " + expressionStr); @@ -449,10 +449,10 @@ public class ExpressionFactory { } else { throw new SolrException(ErrorCode.BAD_REQUEST,"The following function does not exist: " + name); } - + // If the all params are constant, then try to convert the expression to a constant value. expression = expression.convertToConstant(); - + return expression; } @@ -460,7 +460,7 @@ public class ExpressionFactory { * Create an {@link AnalyticsField} out of the given {@link SchemaField}. *

* Currently only fields with doc-values enabled are supported. - * + * * @param field the field to convert for analytics * @return an analytics representation of the field * @throws SolrException if the field is not supported by the analytics framework @@ -556,9 +556,9 @@ public class ExpressionFactory { /** * Get the name of the top function used in the given expression. - * + * * @param expression the expression to find the function name of - * @return the name of the function + * @return the name of the function * @throws SolrException if the expression has incorrect syntax */ private static String getFunctionName(String expression) throws SolrException { @@ -569,10 +569,10 @@ public class ExpressionFactory { String name = m.group(1); return name; } - + /** * Get the params of a function. - * + * * @param function the function to parse * @return an array of param strings * @throws SolrException if the function has incorrect syntax @@ -597,7 +597,7 @@ public class ExpressionFactory { *

  • {@code func('This is " the \\ escaping \' example')} will be treated as {@code func(This is " the \ escaping ' example)} * * In string constants the \ character is used to escape quotes, so it can never be used alone. in order to write a \ you must write \\ - * + * * @param expression the function expression to parse * @param varLengthParamName the name of the variable length parameter that is used in the expression, pass null if none is used. * @param varLengthParamValues the values of the variable length parameter that are used in the expression, pass null if none are used. @@ -610,16 +610,16 @@ public class ExpressionFactory { return new String[0]; } String paramsStr = m.group(1); - + ArrayList paramsList = new ArrayList(); StringBuilder param = new StringBuilder(); - + // Variables to help while filling out the values of for-each lambda functions. boolean inForEach = false; int forEachStart = -1; int forEachIter = -1; int forEachLevel = -1; - + // The current level of nested parenthesis, 0 means the iteration is in no nested parentheses int parenCount = 0; // If the iteration is currently in a single-quote string constant @@ -630,13 +630,13 @@ public class ExpressionFactory { boolean quoteOn = false; // Is the next character escaped. boolean escaped = false; - + char[] chars = paramsStr.toCharArray(); - + // Iterate through every character, building the params one at a time for (int i = 0; i < chars.length; ++i) { char c = chars[i]; - + if (c == ' ' && !quoteOn) { // Ignore white space that is not in string constants continue; @@ -655,12 +655,12 @@ public class ExpressionFactory { } else { paramsList.add(paramStr); } - + param.setLength(0); continue; } else if (c == ',' && !quoteOn && inForEach) { // separate the for each parameters, so they can be replaced with the result of the for each - if (param.charAt(param.length()-1) == variableForEachParam && + if (param.charAt(param.length()-1) == variableForEachParam && (param.charAt(param.length()-2) == '(' || param.charAt(param.length()-2) == ',')) { param.setLength(param.length()-1); param.append(varLengthParamValues[forEachIter++]); @@ -699,7 +699,7 @@ public class ExpressionFactory { throw new SolrException(ErrorCode.BAD_REQUEST,"The following expression has extra end parens: " + param.toString()); } if (inForEach) { - if (param.charAt(param.length()-1) == variableForEachParam && + if (param.charAt(param.length()-1) == variableForEachParam && (param.charAt(param.length()-2) == '(' || param.charAt(param.length()-2) == ',')) { param.setLength(param.length()-1); param.append(varLengthParamValues[forEachIter++]); @@ -774,7 +774,7 @@ public class ExpressionFactory { } return paramsList.toArray(new String[paramsList.size()]); } - + /** * Add the natively supported functionality. */ @@ -811,7 +811,7 @@ public class ExpressionFactory { expressionCreators.put(StringCastFunction.name, StringCastFunction.creatorFunction); expressionCreators.put(SubtractFunction.name, SubtractFunction.creatorFunction); expressionCreators.put(TopFunction.name, TopFunction.creatorFunction); - + // Reduction Functions expressionCreators.put(CountFunction.name, CountFunction.creatorFunction); expressionCreators.put(DocCountFunction.name, DocCountFunction.creatorFunction); @@ -824,7 +824,7 @@ public class ExpressionFactory { expressionCreators.put(PercentileFunction.name, PercentileFunction.creatorFunction); expressionCreators.put(SumFunction.name, SumFunction.creatorFunction); expressionCreators.put(UniqueFunction.name, UniqueFunction.creatorFunction); - + // Variables addSystemVariableFunction(WeightedMeanVariableFunction.name, WeightedMeanVariableFunction.params, WeightedMeanVariableFunction.function); addSystemVariableFunction(SumOfSquaresVariableFunction.name, SumOfSquaresVariableFunction.params, SumOfSquaresVariableFunction.function); diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/AbstractSolrQueryFacet.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/AbstractSolrQueryFacet.java index 230884ad884..56f917aa07b 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/AbstractSolrQueryFacet.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/AbstractSolrQueryFacet.java @@ -31,41 +31,41 @@ import org.apache.solr.search.SolrIndexSearcher; /** * Solr Query Facets are AnalyticsFacets that are calculated after the document streaming phase has occurred in the {@link AnalyticsDriver} - * (during which StreamingFacets and overall expressions are calculated). {@link AbstractSolrQueryFacet}s should not be confused with {@link QueryFacet}s, + * (during which StreamingFacets and overall expressions are calculated). {@link AbstractSolrQueryFacet}s should not be confused with {@link QueryFacet}s, * which are a specific sub-type. - * + * *

    * The filtering for these facets is done through issuing additional Solr queries, and collecting on the resulting documents. * Unlike streaming facets, which have an unspecified amount of facet values (facet buckets), the amount of facet values is determined by the user and * a Solr query is issued for each requested facet value. */ public abstract class AbstractSolrQueryFacet extends AnalyticsFacet { - + protected AbstractSolrQueryFacet(String name) { super(name); } /** * Returns the set of {@link FacetValueQueryExecuter}s, one for each facet value, through the given consumer. - * + * * Each of these executors will be executed after the streaming phase in the {@link AnalyticsDriver}. - * + * * @param filter the overall filter representing the documents being used for the analytics request - * @param queryRequest the queryRequest + * @param queryRequest the queryRequest * @param consumer the consumer of each facet value's executer */ public abstract void createFacetValueExecuters(final Filter filter, SolrQueryRequest queryRequest, Consumer consumer); - + /** * This executer is in charge of issuing the Solr query for a facet value and collecting results as the query is processed. */ public class FacetValueQueryExecuter extends SimpleCollector { private final ReductionDataCollection collection; private final Query query; - + /** * Create an executer to collect the given reduction data from the given Solr query. - * + * * @param collection The reduction data to collect while querying * @param query The query used to filter for the facet value */ @@ -92,7 +92,7 @@ public abstract class AbstractSolrQueryFacet extends AnalyticsFacet { /** * Start the collection for this facet value. - * + * * @param searcher the solr searcher * @throws IOException if an exception occurs during the querying */ diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/AnalyticsFacet.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/AnalyticsFacet.java index d9c0f8cee52..35f1b2c1de1 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/AnalyticsFacet.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/AnalyticsFacet.java @@ -38,28 +38,28 @@ public abstract class AnalyticsFacet { protected final Map reductionData; protected ReductionCollectionManager collectionManager; protected ExpressionCalculator expressionCalculator; - + protected final String name; - + public AnalyticsFacet(String name) { this.reductionData = new LinkedHashMap<>(); this.name = name; } - + /** * Set the {@link ReductionCollectionManager} that manages the collection of the expressions * calculated with this facet. - * + * * @param collectionManager The manager for relevant expressions */ public void setReductionCollectionManager(ReductionCollectionManager collectionManager) { this.collectionManager = collectionManager; } - + /** * Set the {@link ExpressionCalculator} that calculates the collection of the expressions * requested for this facet. - * + * * @param expressionCalculator The calculator for relevant expressions */ public void setExpressionCalculator(ExpressionCalculator expressionCalculator) { @@ -67,9 +67,9 @@ public abstract class AnalyticsFacet { } /** - * Import the shard data from a bit-stream, exported by the {@link #exportShardData} method + * Import the shard data from a bit-stream, exported by the {@link #exportShardData} method * in the each of the collection's shards. - * + * * @param input The bit-stream to import the data from * @throws IOException if an exception occurs while reading from the {@link DataInput} */ @@ -81,7 +81,7 @@ public abstract class AnalyticsFacet { } /** * Import the next facet value's set of {@link ReductionData}. - * + * * @param input the bit-stream to import the reduction data from * @param facetValue the next facet value * @throws IOException if an exception occurs while reading from the input @@ -93,14 +93,14 @@ public abstract class AnalyticsFacet { } else { collectionManager.prepareReductionDataIO(dataCollection); } - + collectionManager.mergeData(); } /** - * Export the shard data through a bit-stream, to be imported by the {@link #importShardData} method + * Export the shard data through a bit-stream, to be imported by the {@link #importShardData} method * in the originating shard. - * + * * @param output The bit-stream to output the data through * @throws IOException if an exception occurs while writing to the {@link DataOutput} */ @@ -112,21 +112,21 @@ public abstract class AnalyticsFacet { } /** * Export the next facet value's set of {@link ReductionData}. - * + * * @param output the bit-stream to output the reduction data to * @param facetValue the next facet value * @throws IOException if an exception occurs while reading from the input */ protected void exportFacetValue(DataOutput output, String facetValue) throws IOException { output.writeUTF(facetValue); - + collectionManager.prepareReductionDataIO(reductionData.get(facetValue)); collectionManager.exportData(); } - + /** * Create the old olap-style response of the facet to be returned in the overall analytics response. - * + * * @return the response of the facet */ public NamedList createOldResponse() { @@ -137,10 +137,10 @@ public abstract class AnalyticsFacet { }); return nl; } - + /** * Create the response of the facet to be returned in the overall analytics response. - * + * * @return the response of the facet */ public Iterable> createResponse() { @@ -154,10 +154,10 @@ public abstract class AnalyticsFacet { }); return list; } - + /** * Get the name of the Facet. This is unique for the grouping. - * + * * @return The name of the Facet */ public String getName() { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/PivotFacet.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/PivotFacet.java index f9e35f7ed6e..d06bba80459 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/PivotFacet.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/PivotFacet.java @@ -27,7 +27,7 @@ import org.apache.solr.analytics.function.ReductionCollectionManager; import org.apache.solr.common.util.NamedList; /** - * A facet that takes in multiple ValueFacet expressions and does analytics calculations over each dimension given. + * A facet that takes in multiple ValueFacet expressions and does analytics calculations over each dimension given. */ public class PivotFacet extends AnalyticsFacet implements StreamingFacet { private final PivotHead pivotHead; @@ -37,7 +37,7 @@ public class PivotFacet extends AnalyticsFacet implements StreamingFacet { super(name); this.pivotHead = new PivotHead(topPivot); } - + @Override public void setReductionCollectionManager(ReductionCollectionManager collectionManager) { pivotHead.setReductionCollectionManager(collectionManager); @@ -62,12 +62,12 @@ public class PivotFacet extends AnalyticsFacet implements StreamingFacet { public void exportShardData(DataOutput output) throws IOException { pivotHead.exportShardData(output); } - + @Override public NamedList createOldResponse() { return new NamedList<>(); } - + @Override public Iterable> createResponse() { return pivotHead.createResponse(); @@ -75,22 +75,22 @@ public class PivotFacet extends AnalyticsFacet implements StreamingFacet { } /** * Typed Pivot class that stores the overall Pivot data and head of the Pivot node chain. - * + * * This class exists so that the {@link PivotFacet} class doesn't have to be typed ( {@code } ). */ class PivotHead implements StreamingFacet { private final PivotNode topPivot; private final Map pivotValues; - + public PivotHead(PivotNode topPivot) { this.topPivot = topPivot; this.pivotValues = new HashMap<>(); } - + public void setReductionCollectionManager(ReductionCollectionManager collectionManager) { topPivot.setReductionCollectionManager(collectionManager); } - + public void setExpressionCalculator(ExpressionCalculator expressionCalculator) { topPivot.setExpressionCalculator(expressionCalculator); } @@ -107,7 +107,7 @@ class PivotHead implements StreamingFacet { public void exportShardData(DataOutput output) throws IOException { topPivot.exportPivot(output, pivotValues); } - + public Iterable> createResponse() { return topPivot.getPivotedResponse(pivotValues); } diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/PivotNode.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/PivotNode.java index c6c0dc496ac..ac3a4ba1abd 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/PivotNode.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/PivotNode.java @@ -55,11 +55,11 @@ public abstract class PivotNode extends SortableFacet implements Consumer extends SortableFacet implements Consumer extends SortableFacet implements Consumer extends SortableFacet implements Consumer> getPivotedResponse(Map pivot); - + /** * A pivot node that has no pivot children. */ @@ -120,7 +120,7 @@ public abstract class PivotNode extends SortableFacet implements Consumer extends SortableFacet implements Consumer extends SortableFacet implements Consumer extends SortableFacet implements Consumer pivotData = currentPivot.get(pivotValue); @@ -226,7 +226,7 @@ public abstract class PivotNode extends SortableFacet implements Consumer pivotData) throws IOException { collectionManager.prepareReductionDataIO(pivotData.pivotReduction); collectionManager.exportData(); - + childPivot.exportPivot(output, pivotData.childPivots); } @@ -251,7 +251,7 @@ public abstract class PivotNode extends SortableFacet implements Consumer queries; - + public QueryFacet(String name, Map queries) { super(name); this.queries = queries; } - + @Override public void createFacetValueExecuters(final Filter filter, SolrQueryRequest queryRequest, Consumer consumer) { queries.forEach( (queryName, query) -> { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/RangeFacet.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/RangeFacet.java index 80e8d21a811..defc9d908b3 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/RangeFacet.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/RangeFacet.java @@ -43,7 +43,7 @@ public class RangeFacet extends AbstractSolrQueryFacet { protected boolean hardEnd = false; protected EnumSet include; protected EnumSet others; - + public RangeFacet(String name, SchemaField field, String start, String end, List gaps) { super(name); this.field = field; @@ -59,8 +59,8 @@ public class RangeFacet extends AbstractSolrQueryFacet { // Computes the end points of the ranges in the rangeFacet final FacetRangeGenerator> rec = FacetRangeGenerator.create(this); final SchemaField sf = field; - - // Create a rangeFacetAccumulator for each range and + + // Create a rangeFacetAccumulator for each range and // collect the documents for that range. for (FacetRange range : rec.getRanges()) { Query q = sf.getType().getRangeQuery(null, sf, range.lower, range.upper, range.includeLower,range.includeUpper); @@ -70,7 +70,7 @@ public class RangeFacet extends AbstractSolrQueryFacet { .add(q, Occur.MUST) .add(filter, Occur.FILTER) .build(); - + ReductionDataCollection dataCol = collectionManager.newDataCollection(); reductionData.put(range.toString(), dataCol); consumer.accept(new FacetValueQueryExecuter(dataCol, rangeQuery)); @@ -112,7 +112,7 @@ public class RangeFacet extends AbstractSolrQueryFacet { public void setOthers(EnumSet others) { this.others = others; } - + public SchemaField getField() { return field; } diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/SortableFacet.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/SortableFacet.java index ef1e04be959..16903aa7b71 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/SortableFacet.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/SortableFacet.java @@ -35,11 +35,11 @@ import com.google.common.collect.Iterables; */ public abstract class SortableFacet extends AnalyticsFacet { protected FacetSortSpecification sort = null; - + protected SortableFacet(String name) { super(name); } - + @Override public NamedList createOldResponse() { final NamedList results = new NamedList<>(); @@ -49,7 +49,7 @@ public abstract class SortableFacet extends AnalyticsFacet { } return results; } - + @Override public Iterable> createResponse() { final LinkedList> results = new LinkedList<>(); @@ -62,20 +62,20 @@ public abstract class SortableFacet extends AnalyticsFacet { } return results; } - + private Iterable getBuckets() { final List facetResults = new ArrayList<>(); reductionData.forEach((facetVal, dataCol) -> { collectionManager.setData(dataCol); facetResults.add(new FacetBucket(facetVal,expressionCalculator.getResults())); }); - + return applyOptions(facetResults); } /** * Apply the sorting options to the given facet results. - * + * * @param facetResults to apply sorting options to * @return the sorted results */ @@ -87,7 +87,7 @@ public abstract class SortableFacet extends AnalyticsFacet { } Comparator comp = sort.getComparator(); Collections.sort(facetResults, comp); - + Iterable facetResultsIter = facetResults; // apply the limit if (sort.getLimit() > 0) { @@ -100,7 +100,7 @@ public abstract class SortableFacet extends AnalyticsFacet { } return facetResultsIter; } - + /** * Specifies how to sort the buckets of a sortable facet. */ @@ -108,7 +108,7 @@ public abstract class SortableFacet extends AnalyticsFacet { private FacetResultsComparator comparator; protected int limit; protected int offset; - + public FacetSortSpecification(FacetResultsComparator comparator, int limit, int offset) { this.comparator = comparator; this.limit = limit; @@ -121,7 +121,7 @@ public abstract class SortableFacet extends AnalyticsFacet { /** * Get the maximum number of buckets to be returned. - * + * * @return the limit */ public int getLimit() { @@ -129,16 +129,16 @@ public abstract class SortableFacet extends AnalyticsFacet { } /** * Set the maximum number of buckets to be returned. - * + * * @param limit the maximum number of buckets */ public void setLimit(int limit) { this.limit = limit; } - + /** * Get the first bucket to return, has to be used with the {@code limit} option. - * + * * @return the bucket offset */ public int getOffset() { @@ -153,24 +153,24 @@ public abstract class SortableFacet extends AnalyticsFacet { public void setSort(SortableFacet.FacetSortSpecification sort) { this.sort = sort; } - + public static class FacetBucket { private final String facetValue; private final Map expressionResults; - + public FacetBucket(String facetValue, Map expressionResults) { this.facetValue = facetValue; this.expressionResults = expressionResults; } - + public Object getResult(String expression) { return expressionResults.get(expression); } - + public Map getResults() { return expressionResults; } - + public String getFacetValue() { return facetValue; } diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/ValueFacet.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/ValueFacet.java index b1d84baf1b4..c7b9cf961bf 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/ValueFacet.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/ValueFacet.java @@ -37,7 +37,7 @@ public class ValueFacet extends SortableFacet implements StreamingFacet, Consume public void addFacetValueCollectionTargets() { expression.streamStrings(this); } - + @Override public void accept(String t) { ReductionDataCollection collection = reductionData.get(t); @@ -48,10 +48,10 @@ public class ValueFacet extends SortableFacet implements StreamingFacet, Consume collectionManager.addCollectTarget(collection); } } - + /** * Get the expression used to create the facet values. - * + * * @return a string mapping expression */ public StringValueStream getExpression() { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/DelegatingComparator.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/DelegatingComparator.java index 200e68b0229..90852d55b6a 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/DelegatingComparator.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/DelegatingComparator.java @@ -27,17 +27,17 @@ import org.apache.solr.common.SolrException.ErrorCode; */ public class DelegatingComparator extends FacetResultsComparator { private final Iterable comparators; - + /** * Create a delegating results comparator. This comparator will in succession use the given comparators, continuing if the values are equal. * Two buckets are considered equal if and only if all comparators find them equal - * + * * @param comparators the comparators to use in succession */ private DelegatingComparator(Iterable comparators) { this.comparators = comparators; } - + public static FacetResultsComparator joinComparators(Collection comparators) throws SolrException { if (comparators.size() == 0) { throw new SolrException(ErrorCode.BAD_REQUEST,"A sort must have at least 1 comparator criteria."); diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/ExpressionComparator.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/ExpressionComparator.java index e4c1940050c..f53b2fdb9d3 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/ExpressionComparator.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/ExpressionComparator.java @@ -23,16 +23,16 @@ import org.apache.solr.analytics.facet.SortableFacet.FacetBucket; */ public class ExpressionComparator> extends FacetResultsComparator { private final String expression; - + /** * Create an entry comparator comparing the given expression. - * + * * @param expression the name of the expression results to compare */ public ExpressionComparator(String expression) { this.expression = expression; } - + @SuppressWarnings("unchecked") public int compare(FacetBucket b1, FacetBucket b2) { T t1 = (T)b1.getResult(expression); diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/FacetResultsComparator.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/FacetResultsComparator.java index 9303f219837..99d94490e49 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/FacetResultsComparator.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/FacetResultsComparator.java @@ -25,26 +25,26 @@ import org.apache.solr.analytics.facet.SortableFacet.FacetBucket; */ public abstract class FacetResultsComparator implements Comparator { protected int resultMult; - + /** * Create a results comparator assuming an ascending ordering. */ public FacetResultsComparator() { setDirection(true); } - + /** * Set the order direction for comparison. - * + * * @param ascending whether to compare using an ascending ordering */ public void setDirection(boolean ascending) { this.resultMult = ascending ? 1 : -1; } - + /** * Compare one facet bucket to another. - * + * * @param b1 the first bucket to compare * @param b2 the second bucket to compare */ diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/FacetValueComparator.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/FacetValueComparator.java index a07ac7b6c53..74fbfe8291a 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/FacetValueComparator.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/FacetValueComparator.java @@ -22,14 +22,14 @@ import org.apache.solr.analytics.facet.SortableFacet.FacetBucket; * A results comparator that compares the name of facet value buckets, which is the string value of the facet value. */ public class FacetValueComparator extends FacetResultsComparator { - + /** * Create a facet value comparator. */ public FacetValueComparator() { super(); } - + @Override public int compare(FacetBucket b1, FacetBucket b2) { return b1.getFacetValue().compareTo(b2.getFacetValue()) * resultMult; diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/package-info.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/package-info.java index c86ad170b3d..b37205ebd37 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/package-info.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/compare/package-info.java @@ -14,8 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -/** + +/** * Comparators used to sort the buckets of an analytics facet. */ package org.apache.solr.analytics.facet.compare; diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/package-info.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/package-info.java index 5812e5482b8..db9514395e4 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/package-info.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/facet/package-info.java @@ -14,8 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -/** + +/** * Facets supported by the analytics component. */ package org.apache.solr.analytics.facet; diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/ExpressionCalculator.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/ExpressionCalculator.java index 3c445554c24..a020c08c557 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/ExpressionCalculator.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/ExpressionCalculator.java @@ -37,11 +37,11 @@ public class ExpressionCalculator { /** * Calculate results for the list of {@link AnalyticsExpression}s. *

    - * NOTE: This method can, and is, called multiple times to generate different responses. + * NOTE: This method can, and is, called multiple times to generate different responses. *
    * The results are determined by which {@link ReductionDataCollection} is passed to the {@link ReductionCollectionManager#setData} * method of the {@link ReductionCollectionManager} managing the reduction for the list of {@link AnalyticsExpression}s. - * + * * @return a {@link NamedList} containing the results */ public Map getResults() { @@ -58,11 +58,11 @@ public class ExpressionCalculator { /** * Calculate results for the list of {@link AnalyticsExpression}s and add them to the given response. *

    - * NOTE: This method can, and is, called multiple times to generate different responses. + * NOTE: This method can, and is, called multiple times to generate different responses. *
    * The results are determined by which {@link ReductionDataCollection} is passed to the {@link ReductionCollectionManager#setData} * method of the {@link ReductionCollectionManager} managing the reduction for the list of {@link AnalyticsExpression}s. - * + * * @param response the response to add the results map to. */ public void addResults(Map response) { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/MergingReductionCollectionManager.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/MergingReductionCollectionManager.java index 1402a769cbf..4f76bd913ab 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/MergingReductionCollectionManager.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/MergingReductionCollectionManager.java @@ -23,20 +23,20 @@ import org.apache.solr.analytics.function.reduction.data.ReductionDataCollector; * The {@link ReductionCollectionManager} used for distributed requests. */ public class MergingReductionCollectionManager extends ReductionCollectionManager { - + public MergingReductionCollectionManager() { super(); } - + public MergingReductionCollectionManager(final ReductionDataCollector[] reductionDataCollectors, final Iterable fields) { super(reductionDataCollectors, fields); } - + @Override protected ReductionCollectionManager createNewManager(final ReductionDataCollector[] reductionDataCollectors, final Iterable fields) { return new MergingReductionCollectionManager(reductionDataCollectors,fields); } - + @Override public void setData(ReductionDataCollection dataCollection) { for (int i = 0; i < reductionDataCollectors.length; i++) { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/ReductionCollectionManager.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/ReductionCollectionManager.java index b3a178cb619..0305937a578 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/ReductionCollectionManager.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/ReductionCollectionManager.java @@ -46,44 +46,44 @@ public class ReductionCollectionManager { private final List> readers; private final List> writers; - + private final Iterable fields; - + public ReductionCollectionManager() { this(new ReductionDataCollector[0], new ArrayList<>(0)); } - + /** * Create a Manager to oversee the given {@link ReductionDataCollector}s. - * + * * @param reductionDataCollectors array of collectors that are collecting over the same set of data * @param fields all fields used by the given collectors */ public ReductionCollectionManager(final ReductionDataCollector[] reductionDataCollectors, final Iterable fields) { this.reductionDataCollectors = reductionDataCollectors; Arrays.sort(reductionDataCollectors, (a,b) -> a.getExpressionStr().compareTo(b.getExpressionStr())); - + reservations = new LinkedList<>(); for (int i = 0; i < reductionDataCollectors.length; i++) { reductionDataCollectors[i].submitReservations(reservation -> reservations.add(reservation)); } - + this.fields = fields; - + this.readers = new ArrayList<>(); this.writers = new ArrayList<>(); } - + /** - * Return whether or not the manager needs collection done, which is false if no collectors are + * Return whether or not the manager needs collection done, which is false if no collectors are * being managed and true if at least one is. - * + * * @return true if at least one collector is being managed */ public boolean needsCollection() { return reductionDataCollectors.length > 0; } - + /** * Merge this collection manager with others. * @@ -93,12 +93,12 @@ public class ReductionCollectionManager { public ReductionCollectionManager merge(Iterable reductionManagers) { HashMap> mergedCollectors = new HashMap<>(); HashMap mergedFields = new HashMap<>(); - + for (ReductionDataCollector collector : reductionDataCollectors) { mergedCollectors.put(collector.getExpressionStr(), collector); } fields.forEach( field -> mergedFields.put(field.getExpressionStr(), field) ); - + reductionManagers.forEach( manager -> { for (ReductionDataCollector collector : manager.reductionDataCollectors) { mergedCollectors.put(collector.getExpressionStr(), collector); @@ -109,10 +109,10 @@ public class ReductionCollectionManager { mergedCollectors.values().toArray(collectors); return createNewManager(collectors, mergedFields.values()); } - + /** * Create an {@link ReductionCollectionManager} to manage the given collectors and fields. - * + * * @param reductionDataCollectors Reduction collectors * @param fields fields used by the reductions * @return a collection manager @@ -120,19 +120,19 @@ public class ReductionCollectionManager { protected ReductionCollectionManager createNewManager(final ReductionDataCollector[] reductionDataCollectors, final Iterable fields) { return new ReductionCollectionManager(reductionDataCollectors,fields); } - + /** * Get the {@link AnalyticsField}s used in the managed expressions. - * + * * @return the fields used */ public Iterable getUsedFields() { return fields; } - + /** * Set the context of the readers of the used {@link AnalyticsField}s. - * + * * @param context the reader context * @throws IOException if an error occurs while setting the fields' context */ @@ -141,10 +141,10 @@ public class ReductionCollectionManager { field.doSetNextReader(context); } } - + /** * Collect the values from the used {@link AnalyticsField}s. - * + * * @param doc the document to collect values for * @throws IOException if an error occurs during field collection */ @@ -153,11 +153,11 @@ public class ReductionCollectionManager { field.collect(doc); } } - + /** * Add a {@link ReductionDataCollection} to target while collecting documents. * This target is valid until the lasting targets are cleared. - * + * * @param target data collection to add document data too */ public void addLastingCollectTarget(ReductionDataCollection target) { @@ -173,11 +173,11 @@ public class ReductionCollectionManager { reductionDataCollectors[i].clearLastingCollectTargets(); } } - + /** * Add a new {@link ReductionDataCollection} to target while collecting the next document. * This target is only valid for the next {@link #apply()} call. - * + * * @return the new data collection being targeted */ public ReductionDataCollection newDataCollectionTarget() { @@ -191,7 +191,7 @@ public class ReductionCollectionManager { /** * Add a {@link ReductionDataCollection} to target while collecting the next document. * This target is only valid for the next {@link #apply()} call. - * + * * @param target data collection to add document data too */ public void addCollectTarget(ReductionDataCollection target) { @@ -199,7 +199,7 @@ public class ReductionCollectionManager { reductionDataCollectors[i].addCollectTarget(target.dataArr[i]); } } - + /** * Apply the values of the collected fields through the expressions' logic to the managed data collectors. * This is called after {@link #collect(int)} has been called and the collection targets have been added. @@ -209,13 +209,13 @@ public class ReductionCollectionManager { reductionDataCollectors[i].collectAndApply();; } } - + /** * Finalize the reductions with the collected data stored in the parameter. - * Once the data is finalized, the {@link ReductionFunction}s that use these - * {@link ReductionDataCollector}s act like regular {@link AnalyticsValue} classes that + * Once the data is finalized, the {@link ReductionFunction}s that use these + * {@link ReductionDataCollector}s act like regular {@link AnalyticsValue} classes that * can be accessed through their {@code get} methods. - * + * * @param dataCollection the collection of reduction data to compute results for */ public void setData(ReductionDataCollection dataCollection) { @@ -223,10 +223,10 @@ public class ReductionCollectionManager { reductionDataCollectors[i].setData(dataCollection.dataArr[i]); } } - + /** * Construct a new data collection holding data for all managed data collectors. - * + * * @return a new data collection */ public ReductionDataCollection newDataCollection() { @@ -237,10 +237,10 @@ public class ReductionCollectionManager { } return newCol; } - + /** * Sets the stream of shard data to merge with. - * + * * @param input the stream of shard data */ public void setShardInput(DataInput input) { @@ -251,7 +251,7 @@ public class ReductionCollectionManager { * Merge the data from the given shard input stream into the set IO data collectors. * Should always be called after {@link #setShardInput(DataInput)} and either {@link #prepareReductionDataIO(ReductionDataCollection)} * or {@link #newDataCollectionIO()} have been called. - * + * * @throws IOException if an error occurs while reading the shard data */ public void mergeData() throws IOException { @@ -259,10 +259,10 @@ public class ReductionCollectionManager { reader.read(); } } - + /** * Sets the stream to export shard data to. - * + * * @param output the stream of shard data */ public void setShardOutput(DataOutput output) { @@ -272,7 +272,7 @@ public class ReductionCollectionManager { /** * Export the data from the set IO data collectors to the given shard output stream. * Should always be called after {@link #setShardOutput(DataOutput)} and {@link #prepareReductionDataIO(ReductionDataCollection)}. - * + * * @throws IOException if an error occurs while writing the shard data */ public void exportData() throws IOException { @@ -280,10 +280,10 @@ public class ReductionCollectionManager { writer.write(); } } - + /** * Set the given data collection to be used for either merging or exporting - * + * * @param col collection to export from or merge to */ public void prepareReductionDataIO(ReductionDataCollection col) { @@ -291,11 +291,11 @@ public class ReductionCollectionManager { reductionDataCollectors[i].dataIO(col.dataArr[i]); } } - + /** * Create a new {@link ReductionDataCollection} to merge to or export from. * Mainly used for creating facet value collectors when merging shard data. - * + * * @return the new data collection created */ public ReductionDataCollection newDataCollectionIO() { @@ -306,10 +306,10 @@ public class ReductionCollectionManager { } return newCol; } - + /** * Holds the collection of {@link ReductionData} that will be updated together. - * + * * For example each grouping will have a separate {@link ReductionDataCollection}, and * ungrouped expressions will have their own as well. */ diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/ReductionFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/ReductionFunction.java index fea01c202bc..0bb347ef772 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/ReductionFunction.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/ReductionFunction.java @@ -29,7 +29,7 @@ public interface ReductionFunction extends AnalyticsValue { /** * Syncs the data collectors with shared versions across the entire Analytics Request * so that as little data as possible is sent across shards. - * + * * @param sync a function that takes in a {@link ReductionDataCollector} and returns a shared version */ void synchronizeDataCollectors(UnaryOperator> sync); diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/AnalyticsField.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/AnalyticsField.java index e0bbb4b2871..df06c355543 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/AnalyticsField.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/AnalyticsField.java @@ -22,19 +22,19 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.solr.analytics.value.AnalyticsValueStream; /** - * An analytics wrapper for Solr Fields. - * + * An analytics wrapper for Solr Fields. + * * Currently only fields with Doc Values enabled can be used in Analytics queries. */ public abstract class AnalyticsField implements AnalyticsValueStream { protected static final int initialArrayLength = 20; - + protected final String fieldName; - + protected AnalyticsField(String fieldName) { this.fieldName = fieldName; } - + @Override public String getExpressionStr() { return fieldName; @@ -44,28 +44,28 @@ public abstract class AnalyticsField implements AnalyticsValueStream { public String getName() { return fieldName; } - + @Override public ExpressionType getExpressionType() { return ExpressionType.FIELD; } - + @Override public AnalyticsValueStream convertToConstant() { return this; } - + /** * Set the segment reader context - * + * * @param context segment context * @throws IOException if an error occurs while loading the leaf reader */ public abstract void doSetNextReader(LeafReaderContext context) throws IOException; - + /** * Collect the value(s) of the wrapped field for the given document, and store the value. - * + * * @param doc ID of the document to collect * @throws IOException if an error occurs while reading the document. */ diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/BooleanField.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/BooleanField.java index 650bc362e14..66921ca4366 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/BooleanField.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/BooleanField.java @@ -40,7 +40,7 @@ public class BooleanField extends AnalyticsField implements CastingBooleanValue public BooleanField(String fieldName) { super(fieldName); } - + @Override public void doSetNextReader(LeafReaderContext context) throws IOException { docValues = DocValues.getSorted(context.reader(), fieldName); @@ -84,7 +84,7 @@ public class BooleanField extends AnalyticsField implements CastingBooleanValue public boolean exists() { return exists; } - + @Override public void streamBooleans(BooleanConsumer cons) { if (exists) { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/BooleanMultiField.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/BooleanMultiField.java index e4eecd3037f..5eeff25b47d 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/BooleanMultiField.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/BooleanMultiField.java @@ -35,7 +35,7 @@ public class BooleanMultiField extends AnalyticsField implements CastingBooleanV private SortedSetDocValues docValues; private int count; private boolean[] values; - + private int trueOrd; public BooleanMultiField(String fieldName) { @@ -43,7 +43,7 @@ public class BooleanMultiField extends AnalyticsField implements CastingBooleanV count = 0; values = new boolean[initialArrayLength]; } - + @Override public void doSetNextReader(LeafReaderContext context) throws IOException { docValues = DocValues.getSortedSet(context.reader(), fieldName); @@ -75,7 +75,7 @@ public class BooleanMultiField extends AnalyticsField implements CastingBooleanV } } } - + private void resizeValues() { boolean[] newValues = new boolean[values.length*2]; for (int i = 0; i < count; ++i) { @@ -83,7 +83,7 @@ public class BooleanMultiField extends AnalyticsField implements CastingBooleanV } values = newValues; } - + @Override public void streamBooleans(BooleanConsumer cons) { for (int i = 0; i < count; ++i) { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DateField.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DateField.java index 88e71bcbc9f..dea541d424f 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DateField.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DateField.java @@ -34,7 +34,7 @@ import org.apache.solr.schema.TrieDateField; * An analytics wrapper for a single-valued {@link TrieDateField} or {@link DatePointField} with DocValues enabled. */ public class DateField extends AnalyticsField implements CastingDateValue { - private NumericDocValues docValues; + private NumericDocValues docValues; private long value; private boolean exists; @@ -75,7 +75,7 @@ public class DateField extends AnalyticsField implements CastingDateValue { public boolean exists() { return exists; } - + @Override public void streamLongs(LongConsumer cons) { if (exists) { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DateMultiPointField.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DateMultiPointField.java index a1560ef77ee..c79a3d3795f 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DateMultiPointField.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DateMultiPointField.java @@ -31,7 +31,7 @@ public class DateMultiPointField extends LongMultiPointField implements CastingD public DateMultiPointField(String fieldName) { super(fieldName); } - + @Override public void streamDates(Consumer cons) { streamLongs(value -> cons.accept(new Date(value))); diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DateMultiTrieField.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DateMultiTrieField.java index cdc77f4ce7c..ae153d90f52 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DateMultiTrieField.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DateMultiTrieField.java @@ -33,7 +33,7 @@ public class DateMultiTrieField extends LongMultiTrieField implements CastingDat public DateMultiTrieField(String fieldName) { super(fieldName); } - + @Override public void streamDates(Consumer cons) { streamLongs(value -> cons.accept(new Date(value))); diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DoubleField.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DoubleField.java index fae26e3e03d..9f4b305a11c 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DoubleField.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DoubleField.java @@ -39,7 +39,7 @@ public class DoubleField extends AnalyticsField implements CastingDoubleValue { public DoubleField(String fieldName) { super(fieldName); } - + @Override public void doSetNextReader(LeafReaderContext context) throws IOException { docValues = DocValues.getNumeric(context.reader(), fieldName); @@ -69,7 +69,7 @@ public class DoubleField extends AnalyticsField implements CastingDoubleValue { public boolean exists() { return exists; } - + @Override public void streamDoubles(DoubleConsumer cons) { if (exists) { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DoubleMultiPointField.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DoubleMultiPointField.java index 0933f6011ba..c0b53e92c0a 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DoubleMultiPointField.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DoubleMultiPointField.java @@ -40,7 +40,7 @@ public class DoubleMultiPointField extends AnalyticsField implements CastingDoub count = 0; values = new double[initialArrayLength]; } - + @Override public void doSetNextReader(LeafReaderContext context) throws IOException { docValues = DocValues.getSortedNumeric(context.reader(), fieldName); @@ -57,13 +57,13 @@ public class DoubleMultiPointField extends AnalyticsField implements CastingDoub count = 0; } } - + private void resizeEmptyValues(int count) { if (count > values.length) { values = new double[count]; } } - + @Override public void streamDoubles(DoubleConsumer cons) { for (int i = 0; i < count; ++i) { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DoubleMultiTrieField.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DoubleMultiTrieField.java index d6ae46911be..9b674aee0bf 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DoubleMultiTrieField.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/DoubleMultiTrieField.java @@ -43,7 +43,7 @@ public class DoubleMultiTrieField extends AnalyticsField implements CastingDoubl count = 0; values = new double[initialArrayLength]; } - + @Override public void doSetNextReader(LeafReaderContext context) throws IOException { docValues = DocValues.getSortedSet(context.reader(), fieldName); @@ -61,7 +61,7 @@ public class DoubleMultiTrieField extends AnalyticsField implements CastingDoubl } } } - + private void resizeValues() { double[] newValues = new double[values.length*2]; for (int i = 0; i < count; ++i) { @@ -69,7 +69,7 @@ public class DoubleMultiTrieField extends AnalyticsField implements CastingDoubl } values = newValues; } - + @Override public void streamDoubles(DoubleConsumer cons) { for (int i = 0; i < count; ++i) { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/FloatField.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/FloatField.java index 82e8295835d..0b1fb6dd079 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/FloatField.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/FloatField.java @@ -40,7 +40,7 @@ public class FloatField extends AnalyticsField implements CastingFloatValue { public FloatField(String fieldName) { super(fieldName); } - + @Override public void doSetNextReader(LeafReaderContext context) throws IOException { docValues = DocValues.getNumeric(context.reader(), fieldName); @@ -74,7 +74,7 @@ public class FloatField extends AnalyticsField implements CastingFloatValue { public boolean exists() { return exists; } - + @Override public void streamFloats(FloatConsumer cons) { if (exists) { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/FloatMultiPointField.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/FloatMultiPointField.java index 947035ea4ca..dd1bd73315f 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/FloatMultiPointField.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/FloatMultiPointField.java @@ -41,12 +41,12 @@ public class FloatMultiPointField extends AnalyticsField implements CastingFloat count = 0; values = new float[initialArrayLength]; } - + @Override public void doSetNextReader(LeafReaderContext context) throws IOException { docValues = DocValues.getSortedNumeric(context.reader(), fieldName); } - + @Override public void collect(int doc) throws IOException { if (docValues.advanceExact(doc)) { @@ -59,13 +59,13 @@ public class FloatMultiPointField extends AnalyticsField implements CastingFloat count = 0; } } - + private void resizeEmptyValues(int count) { if (count > values.length) { values = new float[count]; } } - + @Override public void streamFloats(FloatConsumer cons) { for (int i = 0; i < count; ++i) { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/FloatMultiTrieField.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/FloatMultiTrieField.java index 21b612c0d07..03b5795b888 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/FloatMultiTrieField.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/FloatMultiTrieField.java @@ -44,12 +44,12 @@ public class FloatMultiTrieField extends AnalyticsField implements CastingFloatV count = 0; values = new float[initialArrayLength]; } - + @Override public void doSetNextReader(LeafReaderContext context) throws IOException { docValues = DocValues.getSortedSet(context.reader(), fieldName); } - + @Override public void collect(int doc) throws IOException { count = 0; @@ -63,7 +63,7 @@ public class FloatMultiTrieField extends AnalyticsField implements CastingFloatV } } } - + private void resizeValues() { float[] newValues = new float[values.length*2]; for (int i = 0; i < count; ++i) { @@ -71,7 +71,7 @@ public class FloatMultiTrieField extends AnalyticsField implements CastingFloatV } values = newValues; } - + @Override public void streamFloats(FloatConsumer cons) { for (int i = 0; i < count; ++i) { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/IntField.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/IntField.java index e9ae52b1f6f..cff8afbbab0 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/IntField.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/IntField.java @@ -38,11 +38,11 @@ public class IntField extends AnalyticsField implements CastingIntValue { private NumericDocValues docValues; private int value; private boolean exists; - + public IntField(String fieldName) { super(fieldName); } - + @Override public void doSetNextReader(LeafReaderContext context) throws IOException { docValues = DocValues.getNumeric(context.reader(), fieldName); @@ -84,7 +84,7 @@ public class IntField extends AnalyticsField implements CastingIntValue { public boolean exists() { return exists; } - + @Override public void streamInts(IntConsumer cons) { if (exists) { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/IntMultiPointField.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/IntMultiPointField.java index 2608fa11124..a11a6b76192 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/IntMultiPointField.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/IntMultiPointField.java @@ -42,12 +42,12 @@ public class IntMultiPointField extends AnalyticsField implements CastingIntValu count = 0; values = new int[initialArrayLength]; } - + @Override public void doSetNextReader(LeafReaderContext context) throws IOException { docValues = DocValues.getSortedNumeric(context.reader(), fieldName); } - + @Override public void collect(int doc) throws IOException { if (docValues.advanceExact(doc)) { @@ -60,13 +60,13 @@ public class IntMultiPointField extends AnalyticsField implements CastingIntValu count = 0; } } - + private void resizeEmptyValues(int count) { if (count > values.length) { values = new int[count]; } } - + @Override public void streamInts(IntConsumer cons) { for (int i = 0; i < count; ++i) { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/IntMultiTrieField.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/IntMultiTrieField.java index 699b01d0c70..aa736ce6dee 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/IntMultiTrieField.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/IntMultiTrieField.java @@ -45,12 +45,12 @@ public class IntMultiTrieField extends AnalyticsField implements CastingIntValue count = 0; values = new int[initialArrayLength]; } - + @Override public void doSetNextReader(LeafReaderContext context) throws IOException { docValues = DocValues.getSortedSet(context.reader(), fieldName); } - + @Override public void collect(int doc) throws IOException { count = 0; @@ -64,7 +64,7 @@ public class IntMultiTrieField extends AnalyticsField implements CastingIntValue } } } - + private void resizeValues() { int[] newValues = new int[values.length*2]; for (int i = 0; i < count; ++i) { @@ -72,7 +72,7 @@ public class IntMultiTrieField extends AnalyticsField implements CastingIntValue } values = newValues; } - + @Override public void streamInts(IntConsumer cons) { for (int i = 0; i < count; ++i) { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/LongMultiPointField.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/LongMultiPointField.java index 31d14aea97b..dd0f062372e 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/LongMultiPointField.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/LongMultiPointField.java @@ -40,12 +40,12 @@ public class LongMultiPointField extends AnalyticsField implements CastingLongVa count = 0; values = new long[initialArrayLength]; } - + @Override public void doSetNextReader(LeafReaderContext context) throws IOException { docValues = DocValues.getSortedNumeric(context.reader(), fieldName); } - + @Override public void collect(int doc) throws IOException { if (docValues.advanceExact(doc)) { @@ -58,13 +58,13 @@ public class LongMultiPointField extends AnalyticsField implements CastingLongVa count = 0; } } - + private void resizeEmptyValues(int count) { if (count > values.length) { values = new long[count]; } } - + @Override public void streamLongs(LongConsumer cons) { for (int i = 0; i < count; ++i) { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/LongMultiTrieField.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/LongMultiTrieField.java index dfeca792263..2592c7e66d5 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/LongMultiTrieField.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/LongMultiTrieField.java @@ -43,7 +43,7 @@ public class LongMultiTrieField extends AnalyticsField implements CastingLongVal count = 0; values = new long[initialArrayLength]; } - + @Override public void doSetNextReader(LeafReaderContext context) throws IOException { docValues = DocValues.getSortedSet(context.reader(), fieldName); @@ -61,7 +61,7 @@ public class LongMultiTrieField extends AnalyticsField implements CastingLongVal } } } - + private void resizeValues() { long[] newValues = new long[values.length*2]; for (int i = 0; i < count; ++i) { @@ -69,7 +69,7 @@ public class LongMultiTrieField extends AnalyticsField implements CastingLongVal } values = newValues; } - + @Override public void streamLongs(LongConsumer cons) { for (int i = 0; i < count; ++i) { diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/StringMultiField.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/StringMultiField.java index 39c60f0d711..6e5b7378fef 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/StringMultiField.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/StringMultiField.java @@ -37,7 +37,7 @@ public class StringMultiField extends AnalyticsField implements CastingStringVal super(fieldName); values = new ArrayList<>(initialArrayLength); } - + @Override public void doSetNextReader(LeafReaderContext context) throws IOException { docValues = DocValues.getSortedSet(context.reader(), fieldName); @@ -52,7 +52,7 @@ public class StringMultiField extends AnalyticsField implements CastingStringVal } } } - + @Override public void streamStrings(Consumer cons) { values.forEach(value -> cons.accept(value)); diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/package-info.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/package-info.java index a0e5421582c..8db66760e7f 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/package-info.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/field/package-info.java @@ -14,8 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -/** + +/** * Fields to use for analytics expressions. */ package org.apache.solr.analytics.function.field; diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/AddFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/AddFunction.java index d66f84e71fb..91d150810cd 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/AddFunction.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/AddFunction.java @@ -29,7 +29,7 @@ import org.apache.solr.analytics.value.DoubleValueStream; * Uses: *

      *
    • If a single numeric ValueStream is passed in, a {@link DoubleValue} representing the sum of the values for each document is returned. - *
    • If a numeric ValueStream and a numeric Value are passed in, a {@link DoubleValueStream} representing the sum of + *
    • If a numeric ValueStream and a numeric Value are passed in, a {@link DoubleValueStream} representing the sum of * the Value and each of the values of the ValueStream for a document is returned. * (Or the other way, since the Value and ValueStream can be used in either order) *
    • If multiple numeric Values are passed in, a {@link DoubleValue} representing the sum of all values is returned. @@ -40,13 +40,13 @@ public class AddFunction { public static final CreatorFunction creatorFunction = (params -> { if (params.length == 0) { throw new SolrException(ErrorCode.BAD_REQUEST,"The "+name+" function requires parameters."); - } + } else if (params.length == 1) { if (params[0] instanceof DoubleValueStream) { return LambdaFunction.createDoubleLambdaFunction(name, (a,b) -> a+b, (DoubleValueStream)params[0]); } throw new SolrException(ErrorCode.BAD_REQUEST,"The "+name+" function requires numeric parameters. Incorrect param: "+params[0].getExpressionStr()); - } + } else if (params.length == 2) { AnalyticsValueStream param1 = params[0]; AnalyticsValueStream param2 = params[1]; diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/BottomFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/BottomFunction.java index 10674fddb90..5d33f62bade 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/BottomFunction.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/BottomFunction.java @@ -157,7 +157,7 @@ public class BottomFunction { return LambdaFunction.createStringLambdaFunction(name, (a,b) -> (a.compareTo(b)<0)? a:b, castedParams, false); } } - throw new SolrException(ErrorCode.BAD_REQUEST,"The "+name+" function requires a comparable parameter. " + + throw new SolrException(ErrorCode.BAD_REQUEST,"The "+name+" function requires a comparable parameter. " + "Incorrect parameter: "+params[0].getExpressionStr()); }); } \ No newline at end of file diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/ComparisonFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/ComparisonFunction.java index 1bbf9ae86fa..1ecc930c44c 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/ComparisonFunction.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/ComparisonFunction.java @@ -39,15 +39,15 @@ import org.apache.solr.common.SolrException.ErrorCode; * Uses: *
        *
      • If a two comparable {@link AnalyticsValue}s are passed in, a {@link BooleanValue} representing the comparison of the two values for each document is returned. - *
      • If a comparable {@link AnalyticsValue} and a comparable {@link AnalyticsValueStream} are passed in, + *
      • If a comparable {@link AnalyticsValue} and a comparable {@link AnalyticsValueStream} are passed in, * a {@link BooleanValueStream} representing the comparison of the Value and each of the values of the ValueStream for the document is returned. *
      */ public class ComparisonFunction { - + /** * Create a comparison mapping function, comparing two analytics value (streams) of the same type. - * + * * @param name name of the function * @param comp function to find the result of a comparison * @param params the parameters to compare @@ -84,7 +84,7 @@ public class ComparisonFunction { } throw new SolrException(ErrorCode.BAD_REQUEST,"The "+name+" function requires that at least 1 parameter be single-valued."); } - + /** * A comparison function that tests whether the first parameter is greater than the second parameter */ @@ -137,7 +137,7 @@ public class ComparisonFunction { public static interface CompResultFunction { public boolean apply(int compResult); } - + private static CompResultFunction reverse(CompResultFunction original) { return val -> original.apply(val*-1); } @@ -152,7 +152,7 @@ class CompareDoubleValueFunction extends AbstractBooleanValue { private final String name; private final String funcStr; private final ExpressionType funcType; - + public CompareDoubleValueFunction(String name, DoubleValue exprA, DoubleValue exprB, CompResultFunction comp) { this.name = name; this.exprA = exprA; @@ -198,7 +198,7 @@ class CompareDoubleStreamFunction extends AbstractBooleanValueStream { private final String name; private final String funcStr; private final ExpressionType funcType; - + public CompareDoubleStreamFunction(String name, DoubleValue baseExpr, DoubleValueStream compExpr, CompResultFunction comp) throws SolrException { this.name = name; this.baseExpr = baseExpr; @@ -239,7 +239,7 @@ class CompareDateValueFunction extends AbstractBooleanValue { private final String name; private final String funcStr; private final ExpressionType funcType; - + public CompareDateValueFunction(String name, DateValue exprA, DateValue exprB, CompResultFunction comp) { this.name = name; this.exprA = exprA; @@ -285,7 +285,7 @@ class CompareDateStreamFunction extends AbstractBooleanValueStream { private final String name; private final String funcStr; private final ExpressionType funcType; - + public CompareDateStreamFunction(String name, DateValue baseExpr, DateValueStream compExpr, CompResultFunction comp) throws SolrException { this.name = name; this.baseExpr = baseExpr; diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/ConcatFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/ConcatFunction.java index 9b0809ec728..e4c619fef90 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/ConcatFunction.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/ConcatFunction.java @@ -36,12 +36,12 @@ import org.apache.solr.analytics.value.constant.ConstantValue; *
        *
      • If a single {@link StringValueStream} is passed in, a {@link StringValue} representing the concatenation of the values for each document is returned. * No ordering is guaranteed while concatenating. - *
      • If a {@link StringValue} and a {@link StringValueStream} are passed in, a {@link StringValueStream} representing the concatenation of + *
      • If a {@link StringValue} and a {@link StringValueStream} are passed in, a {@link StringValueStream} representing the concatenation of * the Value and each of the values of the ValueStream for a document is returned. * (Or the other way, since the Value and ValueStream can be used in either order) *
      • If any number (more than 0) of {@link StringValue}s are passed in, a {@link StringValue} representing the concatenation of all values is returned. * If any values don't exist, the overall concatenation value will still exist with an empty string used for any missing values. If none of the parameter - * values exist, then the overall concatenation value will not exist. + * values exist, then the overall concatenation value will not exist. *
      */ public class ConcatFunction { @@ -49,7 +49,7 @@ public class ConcatFunction { public static final CreatorFunction creatorFunction = (params -> { return createConcatFunction(name, name, (a,b) -> a + b, params); }); - + /** * A concatenation mapping function, combining the string values of the given parameters with a given separating string. *
      @@ -77,7 +77,7 @@ public class ConcatFunction { return createConcatFunction(name, uniqueName, (a,b) -> a + sep + b, Arrays.copyOfRange(params, 1, params.length)); }); } - + private static StringValueStream createConcatFunction(String functionName, String uniqueName, TwoStringInStringOutLambda lambda, AnalyticsValueStream[] params) { if (params.length == 0) { throw new SolrException(ErrorCode.BAD_REQUEST, "The "+functionName+" function requires parameters."); diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/DateMathFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/DateMathFunction.java index c9dab98b0cb..8a5756144ab 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/DateMathFunction.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/DateMathFunction.java @@ -73,7 +73,7 @@ class DateMathValueFunction extends AbstractDateValue { public static final String name = DateMathFunction.name; private final String exprStr; private final ExpressionType funcType; - + public DateMathValueFunction(DateValue dateParam, ConstantStringValue mathParam) throws SolrException { this.dateParam = dateParam; this.mathParam = "NOW" + mathParam.getString(); @@ -82,7 +82,7 @@ class DateMathValueFunction extends AbstractDateValue { } private boolean exists = false; - + @Override public long getLong() { Date date = getDate(); @@ -126,7 +126,7 @@ class DateMathStreamFunction extends AbstractDateValueStream { public static final String name = DateMathFunction.name; private final String exprStr; private final ExpressionType funcType; - + public DateMathStreamFunction(DateValueStream dateParam, ConstantStringValue mathParam) throws SolrException { this.dateParam = dateParam; this.mathParam = "NOW" + mathParam.getString(); diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/DateParseFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/DateParseFunction.java index 226acacc9a6..83c0ba7256d 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/DateParseFunction.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/DateParseFunction.java @@ -57,7 +57,7 @@ public class DateParseFunction { return new StringStreamToDateParseFunction((StringValueStream)params[0]); } else { - throw new SolrException(ErrorCode.BAD_REQUEST,"The "+name+" function requires a string or long parameter. " + + throw new SolrException(ErrorCode.BAD_REQUEST,"The "+name+" function requires a string or long parameter. " + "Incorrect parameter: "+params[0].getExpressionStr()); } }); @@ -67,7 +67,7 @@ class LongToDateParseFunction extends AbstractDateValue { public static final String name = DateParseFunction.name; private final String exprStr; private final ExpressionType funcType; - + public LongToDateParseFunction(LongValue param) throws SolrException { this.param = param; this.exprStr = AnalyticsValueStream.createExpressionString(name,param); @@ -101,7 +101,7 @@ class LongStreamToDateParseFunction extends AbstractDateValueStream { public static final String name = DateParseFunction.name; private final String exprStr; private final ExpressionType funcType; - + public LongStreamToDateParseFunction(LongValueStream param) throws SolrException { this.param = param; this.exprStr = AnalyticsValueStream.createExpressionString(name,param); @@ -131,7 +131,7 @@ class StringToDateParseFunction extends AbstractDateValue { public static final String name = DateParseFunction.name; private final String exprStr; private final ExpressionType funcType; - + public StringToDateParseFunction(StringValue param) throws SolrException { this.param = param; this.exprStr = AnalyticsValueStream.createExpressionString(name,param); @@ -176,7 +176,7 @@ class StringStreamToDateParseFunction extends AbstractDateValueStream { public static final String name = DateParseFunction.name; private final String exprStr; private final ExpressionType funcType; - + public StringStreamToDateParseFunction(StringValueStream param) throws SolrException { this.param = param; this.exprStr = AnalyticsValueStream.createExpressionString(name,param); diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/DecimalNumericConversionFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/DecimalNumericConversionFunction.java index df91c956245..c8881ee7b22 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/DecimalNumericConversionFunction.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/DecimalNumericConversionFunction.java @@ -39,15 +39,15 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; /** - * An abstract decimal numeric converting mapping function. For example "round()" would convert a float to an int and a double to a long. + * An abstract decimal numeric converting mapping function. For example "round()" would convert a float to an int and a double to a long. *

      * Takes a numeric Double or Float ValueStream or Value and returns a Long or Int ValueStream or Value, respectively. */ public class DecimalNumericConversionFunction { - + /** * Create a numeric conversion mapping function. - * + * * @param name the name of the function * @param fconv the method to convert floats to ints * @param dconv the method to convert doubles to longs @@ -76,7 +76,7 @@ public class DecimalNumericConversionFunction { throw new SolrException(ErrorCode.BAD_REQUEST,"The "+name+" function requires a numeric parameter."); } } - + /** * A numeric mapping function that returns the floor of the input. */ @@ -86,7 +86,7 @@ public class DecimalNumericConversionFunction { return DecimalNumericConversionFunction.createDecimalConversionFunction(name, val -> (int)Math.floor(val), val -> (long)Math.floor(val), params); }); } - + /** * A numeric mapping function that returns the ceiling of the input. */ @@ -96,7 +96,7 @@ public class DecimalNumericConversionFunction { return DecimalNumericConversionFunction.createDecimalConversionFunction(name, val -> (int)Math.ceil(val), val -> (long)Math.ceil(val), params); }); } - + /** * A numeric mapping function that returns the rounded input. */ @@ -111,7 +111,7 @@ public class DecimalNumericConversionFunction { public static interface ConvertFloatFunction { public int convert(float value); } - + @FunctionalInterface public static interface ConvertDoubleFunction { public long convert(double value); @@ -126,7 +126,7 @@ class ConvertFloatValueFunction extends AbstractIntValue { private final ConvertFloatFunction conv; private final String funcStr; private final ExpressionType funcType; - + public ConvertFloatValueFunction(String name, FloatValue param, ConvertFloatFunction conv) { this.name = name; this.param = param; @@ -166,7 +166,7 @@ class ConvertFloatStreamFunction extends AbstractIntValueStream { private final ConvertFloatFunction conv; private final String funcStr; private final ExpressionType funcType; - + public ConvertFloatStreamFunction(String name, FloatValueStream param, ConvertFloatFunction conv) { this.name = name; this.param = param; @@ -202,7 +202,7 @@ class ConvertDoubleValueFunction extends AbstractLongValue { private final ConvertDoubleFunction conv; private final String funcStr; private final ExpressionType funcType; - + public ConvertDoubleValueFunction(String name, DoubleValue param, ConvertDoubleFunction conv) { this.name = name; this.param = param; @@ -242,7 +242,7 @@ class ConvertDoubleStreamFunction extends AbstractLongValueStream { private final ConvertDoubleFunction conv; private final String funcStr; private final ExpressionType funcType; - + public ConvertDoubleStreamFunction(String name, DoubleValueStream param, ConvertDoubleFunction conv) { this.name = name; this.param = param; diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/DivideFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/DivideFunction.java index e644812d14e..2327004a3e9 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/DivideFunction.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/DivideFunction.java @@ -29,8 +29,8 @@ import org.apache.solr.common.SolrException.ErrorCode; * Uses: *

        *
      • If two numeric Values are passed in, a {@link DoubleValue} representing the divison of the two values is returned. - *
      • If a numeric ValueStream and a numeric Value are passed in, a {@link DoubleValueStream} representing the division of - * the Value and each of the values of the ValueStream for a document is returned. + *
      • If a numeric ValueStream and a numeric Value are passed in, a {@link DoubleValueStream} representing the division of + * the Value and each of the values of the ValueStream for a document is returned. * (Or the other way, since the Value and ValueStream can be used in either order) *
      */ diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/EqualFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/EqualFunction.java index 6e3c651d01e..18a8bce206d 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/EqualFunction.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/EqualFunction.java @@ -34,7 +34,7 @@ import org.apache.solr.common.SolrException.ErrorCode; * Uses: *
        *
      • If two Values are passed in, a {@link BooleanValue} representing the equality of the two values for each document is returned. - *
      • If an {@link AnalyticsValue} and an {@link AnalyticsValueStream} are passed in, + *
      • If an {@link AnalyticsValue} and an {@link AnalyticsValueStream} are passed in, * a {@link BooleanValueStream} representing the equality of the Value and each of the values of the ValueStream for the document is returned. *
      */ @@ -44,10 +44,10 @@ public class EqualFunction { if (params.length != 2) { throw new SolrException(ErrorCode.BAD_REQUEST,"The "+name+" function requires 2 paramaters, " + params.length + " found."); } - + AnalyticsValueStream paramA = params[0]; AnalyticsValueStream paramB = params[1]; - + // Booleans aren't really comparable, so just enable the equal function if (paramA instanceof BooleanValueStream && paramB instanceof BooleanValueStream) { if (paramA instanceof BooleanValue) { @@ -83,7 +83,7 @@ class BooleanValueEqualFunction extends AbstractBooleanValue { public static final String name = EqualFunction.name; private final String funcStr; private final ExpressionType funcType; - + public BooleanValueEqualFunction(BooleanValue exprA, BooleanValue exprB) { this.exprA = exprA; this.exprB = exprB; @@ -126,7 +126,7 @@ class BooleanStreamEqualFunction extends AbstractBooleanValueStream { public static final String name = EqualFunction.name; private final String funcStr; private final ExpressionType funcType; - + public BooleanStreamEqualFunction(BooleanValue baseExpr, BooleanValueStream compExpr) throws SolrException { this.baseExpr = baseExpr; this.compExpr = compExpr; @@ -164,7 +164,7 @@ class ValueEqualFunction extends AbstractBooleanValue { public static final String name = EqualFunction.name; private final String funcStr; private final ExpressionType funcType; - + public ValueEqualFunction(AnalyticsValue exprA, AnalyticsValue exprB) { this.exprA = exprA; this.exprB = exprB; @@ -207,7 +207,7 @@ class StreamEqualFunction extends AbstractBooleanValueStream { public static final String name = EqualFunction.name; private final String funcStr; private final ExpressionType funcType; - + public StreamEqualFunction(AnalyticsValue baseExpr, AnalyticsValueStream compExpr) throws SolrException { this.baseExpr = baseExpr; this.compExpr = compExpr; diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/ExistsFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/ExistsFunction.java index dbdca472e1a..0e289175148 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/ExistsFunction.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/ExistsFunction.java @@ -35,7 +35,7 @@ public class ExistsFunction { public static final CreatorFunction creatorFunction = (params -> { if (params.length != 1) { throw new SolrException(ErrorCode.BAD_REQUEST,"The "+name+" function requires 1 parameter."); - } + } AnalyticsValueStream param = params[0]; if (param instanceof AnalyticsValue) { return new ValueExistsFunction((AnalyticsValue)param); @@ -51,13 +51,13 @@ class ValueStreamExistsFunction extends AbstractBooleanValue { public static final String name = ExistsFunction.name; private final String exprStr; private final ExpressionType funcType; - + public ValueStreamExistsFunction(AnalyticsValueStream param) throws SolrException { this.param = param; this.exprStr = AnalyticsValueStream.createExpressionString(name,param); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,param); } - + private boolean exists; @Override public boolean getBoolean() { @@ -91,13 +91,13 @@ class ValueExistsFunction extends AbstractBooleanValue { public static final String name = ExistsFunction.name; private final String exprStr; private final ExpressionType funcType; - + public ValueExistsFunction(AnalyticsValue param) throws SolrException { this.param = param; this.exprStr = AnalyticsValueStream.createExpressionString(name,param); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,param); } - + @Override public boolean getBoolean() { param.getObject(); diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/FillMissingFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/FillMissingFunction.java index 837bbe6af3c..7bc38fd3fa6 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/FillMissingFunction.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/FillMissingFunction.java @@ -60,7 +60,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; /** - * A mapping function to fill all non-existing values with a given value. + * A mapping function to fill all non-existing values with a given value. *

      * Uses: *

        @@ -135,7 +135,7 @@ class StreamFillMissingFunction extends AbstractAnalyticsValueStream implements public static final String name = FillMissingFunction.name; private final String exprStr; private final ExpressionType funcType; - + public StreamFillMissingFunction(AnalyticsValueStream baseExpr, AnalyticsValueStream fillExpr) throws SolrException { this.baseExpr = baseExpr; this.fillExpr = fillExpr; @@ -160,7 +160,7 @@ class StreamFillMissingFunction extends AbstractAnalyticsValueStream implements exists = true; cons.accept(value); } - + @Override public String getName() { return name; @@ -180,14 +180,14 @@ class ValueFillMissingFunction extends AbstractAnalyticsValue { public static final String name = FillMissingFunction.name; private final String exprStr; private final ExpressionType funcType; - + public ValueFillMissingFunction(AnalyticsValue baseExpr, AnalyticsValue fillExpr) throws SolrException { this.baseExpr = baseExpr; this.fillExpr = fillExpr; this.exprStr = AnalyticsValueStream.createExpressionString(name,baseExpr,fillExpr); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,baseExpr,fillExpr); } - + boolean exists = false; @Override @@ -224,7 +224,7 @@ class BooleanStreamFillMissingFunction extends AbstractBooleanValueStream implem public static final String name = FillMissingFunction.name; private final String exprStr; private final ExpressionType funcType; - + public BooleanStreamFillMissingFunction(BooleanValueStream baseExpr, BooleanValueStream fillExpr) throws SolrException { this.baseExpr = baseExpr; this.fillExpr = fillExpr; @@ -249,7 +249,7 @@ class BooleanStreamFillMissingFunction extends AbstractBooleanValueStream implem exists = true; cons.accept(value); } - + @Override public String getName() { return name; @@ -269,14 +269,14 @@ class BooleanFillMissingFunction extends AbstractBooleanValue { public static final String name = FillMissingFunction.name; private final String exprStr; private final ExpressionType funcType; - + public BooleanFillMissingFunction(BooleanValue baseExpr, BooleanValue fillExpr) throws SolrException { this.baseExpr = baseExpr; this.fillExpr = fillExpr; this.exprStr = AnalyticsValueStream.createExpressionString(name,baseExpr,fillExpr); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,baseExpr,fillExpr); } - + boolean exists = false; @Override @@ -293,7 +293,7 @@ class BooleanFillMissingFunction extends AbstractBooleanValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -313,7 +313,7 @@ class IntStreamFillMissingFunction extends AbstractIntValueStream implements Int public static final String name = FillMissingFunction.name; private final String exprStr; private final ExpressionType funcType; - + public IntStreamFillMissingFunction(IntValueStream baseExpr, IntValueStream fillExpr) throws SolrException { this.baseExpr = baseExpr; this.fillExpr = fillExpr; @@ -338,7 +338,7 @@ class IntStreamFillMissingFunction extends AbstractIntValueStream implements Int exists = true; cons.accept(value); } - + @Override public String getName() { return name; @@ -358,14 +358,14 @@ class IntFillMissingFunction extends AbstractIntValue { public static final String name = FillMissingFunction.name; private final String exprStr; private final ExpressionType funcType; - + public IntFillMissingFunction(IntValue baseExpr, IntValue fillExpr) throws SolrException { this.baseExpr = baseExpr; this.fillExpr = fillExpr; this.exprStr = AnalyticsValueStream.createExpressionString(name,baseExpr,fillExpr); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,baseExpr,fillExpr); } - + boolean exists = false; @Override @@ -382,7 +382,7 @@ class IntFillMissingFunction extends AbstractIntValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -402,7 +402,7 @@ class LongStreamFillMissingFunction extends AbstractLongValueStream implements L public static final String name = FillMissingFunction.name; private final String exprStr; private final ExpressionType funcType; - + public LongStreamFillMissingFunction(LongValueStream baseExpr, LongValueStream fillExpr) throws SolrException { this.baseExpr = baseExpr; this.fillExpr = fillExpr; @@ -427,7 +427,7 @@ class LongStreamFillMissingFunction extends AbstractLongValueStream implements L exists = true; cons.accept(value); } - + @Override public String getName() { return name; @@ -447,14 +447,14 @@ class LongFillMissingFunction extends AbstractLongValue { public static final String name = FillMissingFunction.name; private final String exprStr; private final ExpressionType funcType; - + public LongFillMissingFunction(LongValue baseExpr, LongValue fillExpr) throws SolrException { this.baseExpr = baseExpr; this.fillExpr = fillExpr; this.exprStr = AnalyticsValueStream.createExpressionString(name,baseExpr,fillExpr); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,baseExpr,fillExpr); } - + boolean exists = false; @Override @@ -471,7 +471,7 @@ class LongFillMissingFunction extends AbstractLongValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -491,7 +491,7 @@ class FloatStreamFillMissingFunction extends AbstractFloatValueStream implements public static final String name = FillMissingFunction.name; private final String exprStr; private final ExpressionType funcType; - + public FloatStreamFillMissingFunction(FloatValueStream baseExpr, FloatValueStream fillExpr) throws SolrException { this.baseExpr = baseExpr; this.fillExpr = fillExpr; @@ -516,7 +516,7 @@ class FloatStreamFillMissingFunction extends AbstractFloatValueStream implements exists = true; cons.accept(value); } - + @Override public String getName() { return name; @@ -536,14 +536,14 @@ class FloatFillMissingFunction extends AbstractFloatValue { public static final String name = FillMissingFunction.name; private final String exprStr; private final ExpressionType funcType; - + public FloatFillMissingFunction(FloatValue baseExpr, FloatValue fillExpr) throws SolrException { this.baseExpr = baseExpr; this.fillExpr = fillExpr; this.exprStr = AnalyticsValueStream.createExpressionString(name,baseExpr,fillExpr); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,baseExpr,fillExpr); } - + boolean exists = false; @Override @@ -560,7 +560,7 @@ class FloatFillMissingFunction extends AbstractFloatValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -580,7 +580,7 @@ class DoubleStreamFillMissingFunction extends AbstractDoubleValueStream implemen public static final String name = FillMissingFunction.name; private final String exprStr; private final ExpressionType funcType; - + public DoubleStreamFillMissingFunction(DoubleValueStream baseExpr, DoubleValueStream fillExpr) throws SolrException { this.baseExpr = baseExpr; this.fillExpr = fillExpr; @@ -605,7 +605,7 @@ class DoubleStreamFillMissingFunction extends AbstractDoubleValueStream implemen exists = true; cons.accept(value); } - + @Override public String getName() { return name; @@ -625,14 +625,14 @@ class DoubleFillMissingFunction extends AbstractDoubleValue { public static final String name = FillMissingFunction.name; private final String exprStr; private final ExpressionType funcType; - + public DoubleFillMissingFunction(DoubleValue baseExpr, DoubleValue fillExpr) throws SolrException { this.baseExpr = baseExpr; this.fillExpr = fillExpr; this.exprStr = AnalyticsValueStream.createExpressionString(name,baseExpr,fillExpr); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,baseExpr,fillExpr); } - + boolean exists = false; @Override @@ -649,7 +649,7 @@ class DoubleFillMissingFunction extends AbstractDoubleValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -669,7 +669,7 @@ class DateStreamFillMissingFunction extends AbstractDateValueStream implements L public static final String name = FillMissingFunction.name; private final String exprStr; private final ExpressionType funcType; - + public DateStreamFillMissingFunction(DateValueStream baseExpr, DateValueStream fillExpr) throws SolrException { this.baseExpr = baseExpr; this.fillExpr = fillExpr; @@ -694,7 +694,7 @@ class DateStreamFillMissingFunction extends AbstractDateValueStream implements L exists = true; cons.accept(value); } - + @Override public String getName() { return name; @@ -714,14 +714,14 @@ class DateFillMissingFunction extends AbstractDateValue { public static final String name = FillMissingFunction.name; private final String exprStr; private final ExpressionType funcType; - + public DateFillMissingFunction(DateValue baseExpr, DateValue fillExpr) throws SolrException { this.baseExpr = baseExpr; this.fillExpr = fillExpr; this.exprStr = AnalyticsValueStream.createExpressionString(name,baseExpr,fillExpr); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,baseExpr,fillExpr); } - + boolean exists = false; @Override @@ -758,7 +758,7 @@ class StringStreamFillMissingFunction extends AbstractStringValueStream implemen public static final String name = FillMissingFunction.name; private final String exprStr; private final ExpressionType funcType; - + public StringStreamFillMissingFunction(StringValueStream baseExpr, StringValueStream fillExpr) throws SolrException { this.baseExpr = baseExpr; this.fillExpr = fillExpr; @@ -783,7 +783,7 @@ class StringStreamFillMissingFunction extends AbstractStringValueStream implemen exists = true; cons.accept(value); } - + @Override public String getName() { return name; @@ -803,14 +803,14 @@ class StringFillMissingFunction extends AbstractStringValue { public static final String name = FillMissingFunction.name; private final String exprStr; private final ExpressionType funcType; - + public StringFillMissingFunction(StringValue baseExpr, StringValue fillExpr) throws SolrException { this.baseExpr = baseExpr; this.fillExpr = fillExpr; this.exprStr = AnalyticsValueStream.createExpressionString(name,baseExpr,fillExpr); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,baseExpr,fillExpr); } - + boolean exists = false; @Override @@ -827,7 +827,7 @@ class StringFillMissingFunction extends AbstractStringValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/FilterFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/FilterFunction.java index afd2c2a3ebb..6dac746bf7a 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/FilterFunction.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/FilterFunction.java @@ -60,7 +60,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; /** - * A mapping function to filter a Value or ValueStream. For each document, the value exists if the second parameter + * A mapping function to filter a Value or ValueStream. For each document, the value exists if the second parameter * is true and it doesn't exist otherwise. *

        * The first parameter can be any type of analytics expression. (Required) @@ -135,7 +135,7 @@ class StreamFilterFunction extends AbstractAnalyticsValueStream { public static final String name = FilterFunction.name; private final String exprStr; private final ExpressionType funcType; - + public StreamFilterFunction(AnalyticsValueStream baseExpr, BooleanValue filterExpr) throws SolrException { this.baseExpr = baseExpr; this.filterExpr = filterExpr; @@ -149,7 +149,7 @@ class StreamFilterFunction extends AbstractAnalyticsValueStream { baseExpr.streamObjects(cons); } } - + @Override public String getName() { return name; @@ -169,14 +169,14 @@ class ValueFilterFunction extends AbstractAnalyticsValue { public static final String name = FilterFunction.name; private final String exprStr; private final ExpressionType funcType; - + public ValueFilterFunction(AnalyticsValue baseExpr, BooleanValue filterExpr) throws SolrException { this.baseExpr = baseExpr; this.filterExpr = filterExpr; this.exprStr = AnalyticsValueStream.createExpressionString(name,baseExpr,filterExpr); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,baseExpr,filterExpr); } - + boolean exists = false; @Override @@ -189,7 +189,7 @@ class ValueFilterFunction extends AbstractAnalyticsValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -209,7 +209,7 @@ class BooleanStreamFilterFunction extends AbstractBooleanValueStream { public static final String name = FilterFunction.name; private final String exprStr; private final ExpressionType funcType; - + public BooleanStreamFilterFunction(BooleanValueStream baseExpr, BooleanValue filterExpr) throws SolrException { this.baseExpr = baseExpr; this.filterExpr = filterExpr; @@ -223,7 +223,7 @@ class BooleanStreamFilterFunction extends AbstractBooleanValueStream { baseExpr.streamBooleans(cons); } } - + @Override public String getName() { return name; @@ -243,14 +243,14 @@ class BooleanFilterFunction extends AbstractBooleanValue { public static final String name = FilterFunction.name; private final String exprStr; private final ExpressionType funcType; - + public BooleanFilterFunction(BooleanValue baseExpr, BooleanValue filterExpr) throws SolrException { this.baseExpr = baseExpr; this.filterExpr = filterExpr; this.exprStr = AnalyticsValueStream.createExpressionString(name,baseExpr,filterExpr); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,baseExpr,filterExpr); } - + boolean exists = false; @Override @@ -263,7 +263,7 @@ class BooleanFilterFunction extends AbstractBooleanValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -283,7 +283,7 @@ class IntStreamFilterFunction extends AbstractIntValueStream { public static final String name = FilterFunction.name; private final String exprStr; private final ExpressionType funcType; - + public IntStreamFilterFunction(IntValueStream baseExpr, BooleanValue filterExpr) throws SolrException { this.baseExpr = baseExpr; this.filterExpr = filterExpr; @@ -297,7 +297,7 @@ class IntStreamFilterFunction extends AbstractIntValueStream { baseExpr.streamInts(cons); } } - + @Override public String getName() { return name; @@ -317,14 +317,14 @@ class IntFilterFunction extends AbstractIntValue { public static final String name = FilterFunction.name; private final String exprStr; private final ExpressionType funcType; - + public IntFilterFunction(IntValue baseExpr, BooleanValue filterExpr) throws SolrException { this.baseExpr = baseExpr; this.filterExpr = filterExpr; this.exprStr = AnalyticsValueStream.createExpressionString(name,baseExpr,filterExpr); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,baseExpr,filterExpr); } - + boolean exists = false; @Override @@ -337,7 +337,7 @@ class IntFilterFunction extends AbstractIntValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -357,7 +357,7 @@ class LongStreamFilterFunction extends AbstractLongValueStream { public static final String name = FilterFunction.name; private final String exprStr; private final ExpressionType funcType; - + public LongStreamFilterFunction(LongValueStream baseExpr, BooleanValue filterExpr) throws SolrException { this.baseExpr = baseExpr; this.filterExpr = filterExpr; @@ -371,7 +371,7 @@ class LongStreamFilterFunction extends AbstractLongValueStream { baseExpr.streamLongs(cons); } } - + @Override public String getName() { return name; @@ -391,14 +391,14 @@ class LongFilterFunction extends AbstractLongValue { public static final String name = FilterFunction.name; private final String exprStr; private final ExpressionType funcType; - + public LongFilterFunction(LongValue baseExpr, BooleanValue filterExpr) throws SolrException { this.baseExpr = baseExpr; this.filterExpr = filterExpr; this.exprStr = AnalyticsValueStream.createExpressionString(name,baseExpr,filterExpr); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,baseExpr,filterExpr); } - + boolean exists = false; @Override @@ -411,7 +411,7 @@ class LongFilterFunction extends AbstractLongValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -431,7 +431,7 @@ class FloatStreamFilterFunction extends AbstractFloatValueStream { public static final String name = FilterFunction.name; private final String exprStr; private final ExpressionType funcType; - + public FloatStreamFilterFunction(FloatValueStream baseExpr, BooleanValue filterExpr) throws SolrException { this.baseExpr = baseExpr; this.filterExpr = filterExpr; @@ -445,7 +445,7 @@ class FloatStreamFilterFunction extends AbstractFloatValueStream { baseExpr.streamFloats(cons); } } - + @Override public String getName() { return name; @@ -465,14 +465,14 @@ class FloatFilterFunction extends AbstractFloatValue { public static final String name = FilterFunction.name; private final String exprStr; private final ExpressionType funcType; - + public FloatFilterFunction(FloatValue baseExpr, BooleanValue filterExpr) throws SolrException { this.baseExpr = baseExpr; this.filterExpr = filterExpr; this.exprStr = AnalyticsValueStream.createExpressionString(name,baseExpr,filterExpr); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,baseExpr,filterExpr); } - + boolean exists = false; @Override @@ -485,7 +485,7 @@ class FloatFilterFunction extends AbstractFloatValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -505,7 +505,7 @@ class DoubleStreamFilterFunction extends AbstractDoubleValueStream { public static final String name = FilterFunction.name; private final String exprStr; private final ExpressionType funcType; - + public DoubleStreamFilterFunction(DoubleValueStream baseExpr, BooleanValue filterExpr) throws SolrException { this.baseExpr = baseExpr; this.filterExpr = filterExpr; @@ -519,7 +519,7 @@ class DoubleStreamFilterFunction extends AbstractDoubleValueStream { baseExpr.streamDoubles(cons); } } - + @Override public String getName() { return name; @@ -539,14 +539,14 @@ class DoubleFilterFunction extends AbstractDoubleValue { public static final String name = FilterFunction.name; private final String exprStr; private final ExpressionType funcType; - + public DoubleFilterFunction(DoubleValue baseExpr, BooleanValue filterExpr) throws SolrException { this.baseExpr = baseExpr; this.filterExpr = filterExpr; this.exprStr = AnalyticsValueStream.createExpressionString(name,baseExpr,filterExpr); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,baseExpr,filterExpr); } - + boolean exists = false; @Override @@ -559,7 +559,7 @@ class DoubleFilterFunction extends AbstractDoubleValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -579,7 +579,7 @@ class DateStreamFilterFunction extends AbstractDateValueStream { public static final String name = FilterFunction.name; private final String exprStr; private final ExpressionType funcType; - + public DateStreamFilterFunction(DateValueStream baseExpr, BooleanValue filterExpr) throws SolrException { this.baseExpr = baseExpr; this.filterExpr = filterExpr; @@ -593,7 +593,7 @@ class DateStreamFilterFunction extends AbstractDateValueStream { baseExpr.streamLongs(cons); } } - + @Override public String getName() { return name; @@ -613,14 +613,14 @@ class DateFilterFunction extends AbstractDateValue { public static final String name = FilterFunction.name; private final String exprStr; private final ExpressionType funcType; - + public DateFilterFunction(DateValue baseExpr, BooleanValue filterExpr) throws SolrException { this.baseExpr = baseExpr; this.filterExpr = filterExpr; this.exprStr = AnalyticsValueStream.createExpressionString(name,baseExpr,filterExpr); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,baseExpr,filterExpr); } - + boolean exists = false; @Override @@ -633,7 +633,7 @@ class DateFilterFunction extends AbstractDateValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -653,7 +653,7 @@ class StringStreamFilterFunction extends AbstractStringValueStream { public static final String name = FilterFunction.name; private final String exprStr; private final ExpressionType funcType; - + public StringStreamFilterFunction(StringValueStream baseExpr, BooleanValue filterExpr) throws SolrException { this.baseExpr = baseExpr; this.filterExpr = filterExpr; @@ -667,7 +667,7 @@ class StringStreamFilterFunction extends AbstractStringValueStream { baseExpr.streamStrings(cons); } } - + @Override public String getName() { return name; @@ -687,14 +687,14 @@ class StringFilterFunction extends AbstractStringValue { public static final String name = FilterFunction.name; private final String exprStr; private final ExpressionType funcType; - + public StringFilterFunction(StringValue baseExpr, BooleanValue filterExpr) throws SolrException { this.baseExpr = baseExpr; this.filterExpr = filterExpr; this.exprStr = AnalyticsValueStream.createExpressionString(name,baseExpr,filterExpr); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,baseExpr,filterExpr); } - + boolean exists = false; @Override @@ -707,7 +707,7 @@ class StringFilterFunction extends AbstractStringValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/IfFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/IfFunction.java index 0e4f1b9876a..ff28b905987 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/IfFunction.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/IfFunction.java @@ -60,10 +60,10 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; /** - * An if-else mapping function. + * An if-else mapping function. *

        * Three arguments are required. The first, the conditional parameter, must be a {@link BooleanValue} and - * the later two, the if and else parameters, can be any type of {@link AnalyticsValueStream}. + * the later two, the if and else parameters, can be any type of {@link AnalyticsValueStream}. * For each document, if the conditional value is true then the if-value is used otherwise the else-value is used. *

        * The resulting Value or ValueStream will be typed with the closest super-type of the two non-conditional parameters. @@ -83,10 +83,10 @@ public class IfFunction extends AbstractAnalyticsValueStream { throw new SolrException(ErrorCode.BAD_REQUEST,"The "+name+" function requires 3 paramaters, " + params.length + " found."); } if (!(params[0] instanceof BooleanValue)) { - throw new SolrException(ErrorCode.BAD_REQUEST,"The "+name+" function requires single-valued numeric parameters. " + + throw new SolrException(ErrorCode.BAD_REQUEST,"The "+name+" function requires single-valued numeric parameters. " + "Incorrect parameter: "+params[0].getExpressionStr()); } - + BooleanValue castedIf = (BooleanValue) params[0]; AnalyticsValueStream thenExpr = params[1]; AnalyticsValueStream elseExpr = params[2]; @@ -138,7 +138,7 @@ public class IfFunction extends AbstractAnalyticsValueStream { } return new IfFunction(castedIf,thenExpr,elseExpr); }); - + public IfFunction(BooleanValue ifExpr, AnalyticsValueStream thenExpr, AnalyticsValueStream elseExpr) throws SolrException { this.ifExpr = ifExpr; this.thenExpr = thenExpr; @@ -180,7 +180,7 @@ class ValueIfFunction extends AbstractAnalyticsValue { public static final String name = IfFunction.name; private final String exprStr; private final ExpressionType funcType; - + public ValueIfFunction(BooleanValue ifExpr, AnalyticsValue thenExpr, AnalyticsValue elseExpr) throws SolrException { this.ifExpr = ifExpr; this.thenExpr = thenExpr; @@ -233,7 +233,7 @@ class BooleanStreamIfFunction extends AbstractBooleanValueStream { public static final String name = IfFunction.name; private final String exprStr; private final ExpressionType funcType; - + public BooleanStreamIfFunction(BooleanValue ifExpr, BooleanValueStream thenExpr, BooleanValueStream elseExpr) throws SolrException { this.ifExpr = ifExpr; this.thenExpr = thenExpr; @@ -275,7 +275,7 @@ class BooleanIfFunction extends AbstractBooleanValue { public static final String name = IfFunction.name; private final String exprStr; private final ExpressionType funcType; - + public BooleanIfFunction(BooleanValue ifExpr, BooleanValue thenExpr, BooleanValue elseExpr) throws SolrException { this.ifExpr = ifExpr; this.thenExpr = thenExpr; @@ -328,7 +328,7 @@ class IntStreamIfFunction extends AbstractIntValueStream { public static final String name = IfFunction.name; private final String exprStr; private final ExpressionType funcType; - + public IntStreamIfFunction(BooleanValue ifExpr, IntValueStream thenExpr, IntValueStream elseExpr) throws SolrException { this.ifExpr = ifExpr; this.thenExpr = thenExpr; @@ -370,7 +370,7 @@ class IntIfFunction extends AbstractIntValue { public static final String name = IfFunction.name; private final String exprStr; private final ExpressionType funcType; - + public IntIfFunction(BooleanValue ifExpr, IntValue thenExpr, IntValue elseExpr) throws SolrException { this.ifExpr = ifExpr; this.thenExpr = thenExpr; @@ -423,7 +423,7 @@ class LongStreamIfFunction extends AbstractLongValueStream { public static final String name = IfFunction.name; private final String exprStr; private final ExpressionType funcType; - + public LongStreamIfFunction(BooleanValue ifExpr, LongValueStream thenExpr, LongValueStream elseExpr) throws SolrException { this.ifExpr = ifExpr; this.thenExpr = thenExpr; @@ -465,7 +465,7 @@ class LongIfFunction extends AbstractLongValue { public static final String name = IfFunction.name; private final String exprStr; private final ExpressionType funcType; - + public LongIfFunction(BooleanValue ifExpr, LongValue thenExpr, LongValue elseExpr) throws SolrException { this.ifExpr = ifExpr; this.thenExpr = thenExpr; @@ -518,7 +518,7 @@ class FloatStreamIfFunction extends AbstractFloatValueStream { public static final String name = IfFunction.name; private final String exprStr; private final ExpressionType funcType; - + public FloatStreamIfFunction(BooleanValue ifExpr, FloatValueStream thenExpr, FloatValueStream elseExpr) throws SolrException { this.ifExpr = ifExpr; this.thenExpr = thenExpr; @@ -560,7 +560,7 @@ class FloatIfFunction extends AbstractFloatValue { public static final String name = IfFunction.name; private final String exprStr; private final ExpressionType funcType; - + public FloatIfFunction(BooleanValue ifExpr, FloatValue thenExpr, FloatValue elseExpr) throws SolrException { this.ifExpr = ifExpr; this.thenExpr = thenExpr; @@ -613,7 +613,7 @@ class DoubleStreamIfFunction extends AbstractDoubleValueStream { public static final String name = IfFunction.name; private final String exprStr; private final ExpressionType funcType; - + public DoubleStreamIfFunction(BooleanValue ifExpr, DoubleValueStream thenExpr, DoubleValueStream elseExpr) throws SolrException { this.ifExpr = ifExpr; this.thenExpr = thenExpr; @@ -655,7 +655,7 @@ class DoubleIfFunction extends AbstractDoubleValue { public static final String name = IfFunction.name; private final String exprStr; private final ExpressionType funcType; - + public DoubleIfFunction(BooleanValue ifExpr, DoubleValue thenExpr, DoubleValue elseExpr) throws SolrException { this.ifExpr = ifExpr; this.thenExpr = thenExpr; @@ -708,7 +708,7 @@ class DateStreamIfFunction extends AbstractDateValueStream { public static final String name = IfFunction.name; private final String exprStr; private final ExpressionType funcType; - + public DateStreamIfFunction(BooleanValue ifExpr, DateValueStream thenExpr, DateValueStream elseExpr) throws SolrException { this.ifExpr = ifExpr; this.thenExpr = thenExpr; @@ -750,7 +750,7 @@ class DateIfFunction extends AbstractDateValue { public static final String name = IfFunction.name; private final String exprStr; private final ExpressionType funcType; - + public DateIfFunction(BooleanValue ifExpr, DateValue thenExpr, DateValue elseExpr) throws SolrException { this.ifExpr = ifExpr; this.thenExpr = thenExpr; @@ -803,7 +803,7 @@ class StringStreamIfFunction extends AbstractStringValueStream { public static final String name = IfFunction.name; private final String exprStr; private final ExpressionType funcType; - + public StringStreamIfFunction(BooleanValue ifExpr, StringValueStream thenExpr, StringValueStream elseExpr) throws SolrException { this.ifExpr = ifExpr; this.thenExpr = thenExpr; @@ -824,7 +824,7 @@ class StringStreamIfFunction extends AbstractStringValueStream { } } } - + @Override public String getName() { return name; @@ -845,7 +845,7 @@ class StringIfFunction extends AbstractStringValue { public static final String name = IfFunction.name; private final String exprStr; private final ExpressionType funcType; - + public StringIfFunction(BooleanValue ifExpr, StringValue thenExpr, StringValue elseExpr) throws SolrException { this.ifExpr = ifExpr; this.thenExpr = thenExpr; diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/LambdaFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/LambdaFunction.java index ff8a1a6502c..fcaf332b899 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/LambdaFunction.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/LambdaFunction.java @@ -103,17 +103,17 @@ import org.apache.solr.common.SolrException.ErrorCode; */ public class LambdaFunction { private static final boolean defaultMultiExistsMethod = true; - + /* ********************* - * + * * Boolean Functions - * + * * *********************/ - + /** * Creates a function that takes in either a single or multi valued boolean expression and returns the same type of expression with * the given lambda function applied to every value. - * + * * @param name name for the function * @param lambda the function to be applied to every value: {@code (boolean) -> boolean} * @param param the expression to apply the lambda to @@ -128,8 +128,8 @@ public class LambdaFunction { } /** * Creates a function that takes in a multi-valued boolean expression and returns a single-valued boolean expression. - * The given lambda is used to associatively (order not guaranteed) reduce all values for a document down to a single value. - * + * The given lambda is used to associatively (order not guaranteed) reduce all values for a document down to a single value. + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (boolean, boolean) -> boolean} * @param param the expression to be reduced per-document @@ -149,7 +149,7 @@ public class LambdaFunction { * The inputs can be either {@code func(single,multi)} or {@code func(multi,single)}. *

      *

      - * + * * @param name name for the function * @param lambda the function to be applied to every value: {@code (boolean,boolean) -> boolean} * @param param1 the first parameter in the lambda @@ -171,7 +171,7 @@ public class LambdaFunction { /** * Forwards the creation of the function to {@link #createBooleanLambdaFunction(String, TwoBoolInBoolOutLambda, BooleanValue[], boolean)}, * using {@value #defaultMultiExistsMethod} for the last argument ({@code allMustExist}). - * + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (boolean, boolean) -> boolean} * @param params the expressions to reduce @@ -186,7 +186,7 @@ public class LambdaFunction { *
      * For a document, every parameter's value must exist for the resulting value to exist if {@code allMustExist} is true. * If {@code allMustExist} is false, only one of the parameters' values must exist. - * + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (boolean, boolean) -> boolean} * @param params the expressions to reduce @@ -200,17 +200,17 @@ public class LambdaFunction { return new MultiBooleanValueInBooleanValueOutRequireOneFunction(name,lambda,params); } } - + /* ********************* - * + * * Integer Functions - * + * * *********************/ - + /** * Creates a function that takes in either a single or multi valued integer expression and returns the same type of expression with * the given lambda function applied to every value. - * + * * @param name name for the function * @param lambda the function to be applied to every value: {@code (integer) -> integer} * @param param the expression to apply the lambda to @@ -225,8 +225,8 @@ public class LambdaFunction { } /** * Creates a function that takes in a multi-valued integer expression and returns a single-valued integer expression. - * The given lambda is used to associatively (order not guaranteed) reduce all values for a document down to a single value. - * + * The given lambda is used to associatively (order not guaranteed) reduce all values for a document down to a single value. + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (integer, integer) -> integer} * @param param the expression to be reduced per-document @@ -246,7 +246,7 @@ public class LambdaFunction { * The inputs can be either {@code func(single,multi)} or {@code func(multi,single)}. *

    *

    - * + * * @param name name for the function * @param lambda the function to be applied to every value: {@code (integer,integer) -> integer} * @param param1 the first parameter in the lambda @@ -268,7 +268,7 @@ public class LambdaFunction { /** * Forwards the creation of the function to {@link #createIntLambdaFunction(String, TwoIntInIntOutLambda, IntValue[], boolean)}, * using {@value #defaultMultiExistsMethod} for the last argument ({@code allMustExist}). - * + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (boolean, boolean) -> boolean} * @param params the expressions to reduce @@ -283,7 +283,7 @@ public class LambdaFunction { *
    * For a document, every parameter's value must exist for the resulting value to exist if {@code allMustExist} is true. * If {@code allMustExist} is false, only one of the parameters' values must exist. - * + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (integer, integer) -> integer} * @param params the expressions to reduce @@ -297,17 +297,17 @@ public class LambdaFunction { return new MultiIntValueInIntValueOutRequireOneFunction(name,lambda,params); } } - + /* ********************* - * + * * Long Functions - * + * * *********************/ - + /** * Creates a function that takes in either a single or multi valued long expression and returns the same type of expression with * the given lambda function applied to every value. - * + * * @param name name for the function * @param lambda the function to be applied to every value: {@code (long) -> long} * @param param the expression to apply the lambda to @@ -322,8 +322,8 @@ public class LambdaFunction { } /** * Creates a function that takes in a multi-valued long expression and returns a single-valued long expression. - * The given lambda is used to associatively (order not guaranteed) reduce all values for a document down to a single value. - * + * The given lambda is used to associatively (order not guaranteed) reduce all values for a document down to a single value. + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (boolean, boolean) -> boolean} * @param param the expression to be reduced per-document @@ -343,7 +343,7 @@ public class LambdaFunction { * The inputs can be either {@code func(single,multi)} or {@code func(multi,single)}. * *

    - * + * * @param name name for the function * @param lambda the function to be applied to every value: {@code (long,long) -> long} * @param param1 the first parameter in the lambda @@ -365,7 +365,7 @@ public class LambdaFunction { /** * Forwards the creation of the function to {@link #createLongLambdaFunction(String, TwoLongInLongOutLambda, LongValue[], boolean)}, * using {@value #defaultMultiExistsMethod} for the last argument ({@code allMustExist}). - * + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (boolean, boolean) -> boolean} * @param params the expressions to reduce @@ -380,7 +380,7 @@ public class LambdaFunction { *
    * For a document, every parameter's value must exist for the resulting value to exist if {@code allMustExist} is true. * If {@code allMustExist} is false, only one of the parameters' values must exist. - * + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (long, long) -> long} * @param params the expressions to reduce @@ -394,17 +394,17 @@ public class LambdaFunction { return new MultiLongValueInLongValueOutRequireOneFunction(name,lambda,params); } } - + /* ********************* - * + * * Float Functions - * + * * *********************/ - + /** * Creates a function that takes in either a single or multi valued float expression and returns the same type of expression with * the given lambda function applied to every value. - * + * * @param name name for the function * @param lambda the function to be applied to every value: {@code (float) -> float} * @param param the expression to apply the lambda to @@ -419,8 +419,8 @@ public class LambdaFunction { } /** * Creates a function that takes in a multi-valued float expression and returns a single-valued float expression. - * The given lambda is used to associatively (order not guaranteed) reduce all values for a document down to a single value. - * + * The given lambda is used to associatively (order not guaranteed) reduce all values for a document down to a single value. + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (float, float) -> float} * @param param the expression to be reduced per-document @@ -440,7 +440,7 @@ public class LambdaFunction { * The inputs can be either {@code func(single,multi)} or {@code func(multi,single)}. * *

    - * + * * @param name name for the function * @param lambda the function to be applied to every value: {@code (float,float) -> float} * @param param1 the first parameter in the lambda @@ -462,7 +462,7 @@ public class LambdaFunction { /** * Forwards the creation of the function to {@link #createFloatLambdaFunction(String, TwoFloatInFloatOutLambda, FloatValue[], boolean)}, * using {@value #defaultMultiExistsMethod} for the last argument ({@code allMustExist}). - * + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (boolean, boolean) -> boolean} * @param params the expressions to reduce @@ -477,7 +477,7 @@ public class LambdaFunction { *
    * For a document, every parameter's value must exist for the resulting value to exist if {@code allMustExist} is true. * If {@code allMustExist} is false, only one of the parameters' values must exist. - * + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (float, float) -> float} * @param params the expressions to reduce @@ -491,17 +491,17 @@ public class LambdaFunction { return new MultiFloatValueInFloatValueOutRequireOneFunction(name,lambda,params); } } - + /* ********************* - * + * * Double Functions - * + * * *********************/ - + /** * Creates a function that takes in either a single or multi valued double expression and returns the same type of expression with * the given lambda function applied to every value. - * + * * @param name name for the function * @param lambda the function to be applied to every value: {@code (double) -> double} * @param param the expression to apply the lambda to @@ -516,8 +516,8 @@ public class LambdaFunction { } /** * Creates a function that takes in a multi-valued double expression and returns a single-valued double expression. - * The given lambda is used to associatively (order not guaranteed) reduce all values for a document down to a single value. - * + * The given lambda is used to associatively (order not guaranteed) reduce all values for a document down to a single value. + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (double, double) -> double} * @param param the expression to be reduced per-document @@ -537,7 +537,7 @@ public class LambdaFunction { * The inputs can be either {@code func(single,multi)} or {@code func(multi,single)}. * *

    - * + * * @param name name for the function * @param lambda the function to be applied to every value: {@code (double,double) -> double} * @param param1 the first parameter in the lambda @@ -559,7 +559,7 @@ public class LambdaFunction { /** * Forwards the creation of the function to {@link #createDoubleLambdaFunction(String, TwoDoubleInDoubleOutLambda, DoubleValue[], boolean)}, * using {@value #defaultMultiExistsMethod} for the last argument ({@code allMustExist}). - * + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (boolean, boolean) -> boolean} * @param params the expressions to reduce @@ -574,7 +574,7 @@ public class LambdaFunction { *
    * For a document, every parameter's value must exist for the resulting value to exist if {@code allMustExist} is true. * If {@code allMustExist} is false, only one of the parameters' values must exist. - * + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (double, double) -> double} * @param params the expressions to reduce @@ -588,20 +588,20 @@ public class LambdaFunction { return new MultiDoubleValueInDoubleValueOutRequireOneFunction(name,lambda,params); } } - + /* ********************* - * + * * Date Functions - * + * * *********************/ - + /** * Creates a function that takes in either a single or multi valued date expression and returns the same type of expression with - * the given lambda function applied to every value. - * + * the given lambda function applied to every value. + * *

    * NOTE: The lambda must work on longs, not Date objects - * + * * @param name name for the function * @param lambda the function to be applied to every value: {@code (long) -> long} * @param param the expression to apply the lambda to @@ -616,11 +616,11 @@ public class LambdaFunction { } /** * Creates a function that takes in a multi-valued date expression and returns a single-valued date expression. - * The given lambda is used to associatively (order not guaranteed) reduce all values for a document down to a single value. - * + * The given lambda is used to associatively (order not guaranteed) reduce all values for a document down to a single value. + * *

    * NOTE: The lambda must work on longs, not Date objects - * + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (long, long) -> long} * @param param the expression to be reduced per-document @@ -639,10 +639,10 @@ public class LambdaFunction { *
    * The inputs can be either {@code func(single,multi)} or {@code func(multi,single)}. * - * + * *

    * NOTE: The lambda must work on longs, not Date objects - * + * * @param name name for the function * @param lambda the function to be applied to every value: {@code (long,long) -> long} * @param param1 the first parameter in the lambda @@ -664,7 +664,7 @@ public class LambdaFunction { /** * Forwards the creation of the function to {@link #createDateLambdaFunction(String, TwoLongInLongOutLambda, DateValue[], boolean)}, * using {@value #defaultMultiExistsMethod} for the last argument ({@code allMustExist}). - * + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (boolean, boolean) -> boolean} * @param params the expressions to reduce @@ -679,10 +679,10 @@ public class LambdaFunction { *
    * For a document, every parameter's value must exist for the resulting value to exist if {@code allMustExist} is true. * If {@code allMustExist} is false, only one of the parameters' values must exist. - * + * *

    * NOTE: The lambda must work on longs, not Date objects - * + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (long, long) -> long} * @param params the expressions to reduce @@ -696,17 +696,17 @@ public class LambdaFunction { return new MultiDateValueInDateValueOutRequireOneFunction(name,lambda,params); } } - + /* ********************* - * + * * String Functions - * + * * *********************/ - + /** * Creates a function that takes in either a single or multi valued string expression and returns the same type of expression with * the given lambda function applied to every value. - * + * * @param name name for the function * @param lambda the function to be applied to every value: {@code (String) -> String} * @param param the expression to apply the lambda to @@ -721,8 +721,8 @@ public class LambdaFunction { } /** * Creates a function that takes in a multi-valued string expression and returns a single-valued string expression. - * The given lambda is used to associatively (order not guaranteed) reduce all values for a document down to a single value. - * + * The given lambda is used to associatively (order not guaranteed) reduce all values for a document down to a single value. + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (String, String) -> String} * @param param the expression to be reduced per-document @@ -742,7 +742,7 @@ public class LambdaFunction { * The inputs can be either {@code func(single,multi)} or {@code func(multi,single)}. * *

    - * + * * @param name name for the function * @param lambda the function to be applied to every value: {@code (String,String) -> String} * @param param1 the first parameter in the lambda @@ -764,7 +764,7 @@ public class LambdaFunction { /** * Forwards the creation of the function to {@link #createStringLambdaFunction(String, TwoStringInStringOutLambda, StringValue[], boolean)}, * using {@value #defaultMultiExistsMethod} for the last argument ({@code allMustExist}). - * + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (boolean, boolean) -> boolean} * @param params the expressions to reduce @@ -779,7 +779,7 @@ public class LambdaFunction { *
    * For a document, every parameter's value must exist for the resulting value to exist if {@code allMustExist} is true. * If {@code allMustExist} is false, only one of the parameters' values must exist. - * + * * @param name name for the function * @param lambda the associative function used to reduce the values: {@code (String, String) -> String} * @param params the expressions to reduce @@ -793,7 +793,7 @@ public class LambdaFunction { return new MultiStringValueInStringValueOutRequireOneFunction(name,lambda,params); } } - + /* * Single Parameter @@ -876,7 +876,7 @@ public class LambdaFunction { public static interface DoubleInStringOutLambda { String apply(double a); } @FunctionalInterface public static interface StringInStringOutLambda { String apply(String a); } - + /* * Two Parameters */ @@ -958,14 +958,14 @@ public class LambdaFunction { public static interface TwoDoubleInStringOutLambda { String apply(double a, double b); } @FunctionalInterface public static interface TwoStringInStringOutLambda { String apply(String a, String b); } -} +} class BooleanValueInBooleanValueOutFunction extends AbstractBooleanValue { private final BooleanValue param; private final BoolInBoolOutLambda lambda; private final String name; private final String exprStr; private final ExpressionType funcType; - + public BooleanValueInBooleanValueOutFunction(String name, BoolInBoolOutLambda lambda, BooleanValue param) { this.name = name; this.lambda = lambda; @@ -986,7 +986,7 @@ class BooleanValueInBooleanValueOutFunction extends AbstractBooleanValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -1006,7 +1006,7 @@ class BooleanStreamInBooleanStreamOutFunction extends AbstractBooleanValueStream private final String name; private final String exprStr; private final ExpressionType funcType; - + public BooleanStreamInBooleanStreamOutFunction(String name, BoolInBoolOutLambda lambda, BooleanValueStream param) { this.name = name; this.lambda = lambda; @@ -1019,7 +1019,7 @@ class BooleanStreamInBooleanStreamOutFunction extends AbstractBooleanValueStream public void streamBooleans(BooleanConsumer cons) { param.streamBooleans(value -> cons.accept(lambda.apply(value))); } - + @Override public String getName() { return name; @@ -1039,7 +1039,7 @@ class BooleanStreamInBooleanValueOutFunction extends AbstractBooleanValue implem private final String name; private final String exprStr; private final ExpressionType funcType; - + public BooleanStreamInBooleanValueOutFunction(String name, TwoBoolInBoolOutLambda lambda, BooleanValueStream param) { this.name = name; this.lambda = lambda; @@ -1069,7 +1069,7 @@ class BooleanStreamInBooleanValueOutFunction extends AbstractBooleanValue implem value = lambda.apply(value, paramValue); } } - + @Override public String getName() { return name; @@ -1090,7 +1090,7 @@ class TwoBooleanValueInBooleanValueOutFunction extends AbstractBooleanValue { private final String name; private final String exprStr; private final ExpressionType funcType; - + public TwoBooleanValueInBooleanValueOutFunction(String name, TwoBoolInBoolOutLambda lambda, BooleanValue param1, BooleanValue param2) { this.name = name; this.lambda = lambda; @@ -1099,7 +1099,7 @@ class TwoBooleanValueInBooleanValueOutFunction extends AbstractBooleanValue { this.exprStr = AnalyticsValueStream.createExpressionString(name,param1,param2); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,param1,param2); } - + private boolean exists = false; @Override @@ -1112,7 +1112,7 @@ class TwoBooleanValueInBooleanValueOutFunction extends AbstractBooleanValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -1133,7 +1133,7 @@ class BooleanValueBooleanStreamInBooleanStreamOutFunction extends AbstractBoolea private final String name; private final String exprStr; private final ExpressionType funcType; - + public BooleanValueBooleanStreamInBooleanStreamOutFunction(String name, TwoBoolInBoolOutLambda lambda, BooleanValue param1, BooleanValueStream param2) { this.name = name; this.lambda = lambda; @@ -1171,7 +1171,7 @@ class BooleanStreamBooleanValueInBooleanStreamOutFunction extends AbstractBoolea private final String name; private final String exprStr; private final ExpressionType funcType; - + public BooleanStreamBooleanValueInBooleanStreamOutFunction(String name, TwoBoolInBoolOutLambda lambda, BooleanValueStream param1, BooleanValue param2) { this.name = name; this.lambda = lambda; @@ -1208,7 +1208,7 @@ abstract class MultiBooleanValueInBooleanValueOutFunction extends AbstractBoolea private final String name; private final String exprStr; private final ExpressionType funcType; - + public MultiBooleanValueInBooleanValueOutFunction(String name, TwoBoolInBoolOutLambda lambda, BooleanValue[] params) { this.name = name; this.lambda = lambda; @@ -1216,14 +1216,14 @@ abstract class MultiBooleanValueInBooleanValueOutFunction extends AbstractBoolea this.exprStr = AnalyticsValueStream.createExpressionString(name,params); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,params); } - + protected boolean exists = false; protected boolean temp; @Override public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -1238,11 +1238,11 @@ abstract class MultiBooleanValueInBooleanValueOutFunction extends AbstractBoolea } } class MultiBooleanValueInBooleanValueOutRequireAllFunction extends MultiBooleanValueInBooleanValueOutFunction { - + public MultiBooleanValueInBooleanValueOutRequireAllFunction(String name, TwoBoolInBoolOutLambda lambda, BooleanValue[] params) { super(name, lambda, params); } - + @Override public boolean getBoolean() { boolean value = params[0].getBoolean(); @@ -1255,11 +1255,11 @@ class MultiBooleanValueInBooleanValueOutRequireAllFunction extends MultiBooleanV } } class MultiBooleanValueInBooleanValueOutRequireOneFunction extends MultiBooleanValueInBooleanValueOutFunction { - + public MultiBooleanValueInBooleanValueOutRequireOneFunction(String name, TwoBoolInBoolOutLambda lambda, BooleanValue[] params) { super(name, lambda, params); } - + @Override public boolean getBoolean() { int i = -1; @@ -1287,7 +1287,7 @@ class IntValueInIntValueOutFunction extends AbstractIntValue { private final String name; private final String exprStr; private final ExpressionType funcType; - + public IntValueInIntValueOutFunction(String name, IntInIntOutLambda lambda, IntValue param) { this.name = name; this.lambda = lambda; @@ -1308,7 +1308,7 @@ class IntValueInIntValueOutFunction extends AbstractIntValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -1328,7 +1328,7 @@ class IntStreamInIntStreamOutFunction extends AbstractIntValueStream { private final String name; private final String exprStr; private final ExpressionType funcType; - + public IntStreamInIntStreamOutFunction(String name, IntInIntOutLambda lambda, IntValueStream param) { this.name = name; this.lambda = lambda; @@ -1341,7 +1341,7 @@ class IntStreamInIntStreamOutFunction extends AbstractIntValueStream { public void streamInts(IntConsumer cons) { param.streamInts(value -> cons.accept(lambda.apply(value))); } - + @Override public String getName() { return name; @@ -1361,7 +1361,7 @@ class IntStreamInIntValueOutFunction extends AbstractIntValue implements IntCons private final String name; private final String exprStr; private final ExpressionType funcType; - + public IntStreamInIntValueOutFunction(String name, TwoIntInIntOutLambda lambda, IntValueStream param) { this.name = name; this.lambda = lambda; @@ -1391,7 +1391,7 @@ class IntStreamInIntValueOutFunction extends AbstractIntValue implements IntCons value = lambda.apply(value, paramValue); } } - + @Override public String getName() { return name; @@ -1412,7 +1412,7 @@ class TwoIntValueInIntValueOutFunction extends AbstractIntValue { private final String name; private final String exprStr; private final ExpressionType funcType; - + public TwoIntValueInIntValueOutFunction(String name, TwoIntInIntOutLambda lambda, IntValue param1, IntValue param2) { this.name = name; this.lambda = lambda; @@ -1421,7 +1421,7 @@ class TwoIntValueInIntValueOutFunction extends AbstractIntValue { this.exprStr = AnalyticsValueStream.createExpressionString(name,param1,param2); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,param1,param2); } - + private boolean exists = false; @Override @@ -1434,7 +1434,7 @@ class TwoIntValueInIntValueOutFunction extends AbstractIntValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -1455,7 +1455,7 @@ class IntValueIntStreamInIntStreamOutFunction extends AbstractIntValueStream { private final String name; private final String exprStr; private final ExpressionType funcType; - + public IntValueIntStreamInIntStreamOutFunction(String name, TwoIntInIntOutLambda lambda, IntValue param1, IntValueStream param2) { this.name = name; this.lambda = lambda; @@ -1493,7 +1493,7 @@ class IntStreamIntValueInIntStreamOutFunction extends AbstractIntValueStream { private final String name; private final String exprStr; private final ExpressionType funcType; - + public IntStreamIntValueInIntStreamOutFunction(String name, TwoIntInIntOutLambda lambda, IntValueStream param1, IntValue param2) { this.name = name; this.lambda = lambda; @@ -1530,7 +1530,7 @@ abstract class MultiIntValueInIntValueOutFunction extends AbstractIntValue { private final String name; private final String exprStr; private final ExpressionType funcType; - + public MultiIntValueInIntValueOutFunction(String name, TwoIntInIntOutLambda lambda, IntValue[] params) { this.name = name; this.lambda = lambda; @@ -1538,14 +1538,14 @@ abstract class MultiIntValueInIntValueOutFunction extends AbstractIntValue { this.exprStr = AnalyticsValueStream.createExpressionString(name,params); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,params); } - + protected boolean exists = false; protected int temp; @Override public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -1560,11 +1560,11 @@ abstract class MultiIntValueInIntValueOutFunction extends AbstractIntValue { } } class MultiIntValueInIntValueOutRequireAllFunction extends MultiIntValueInIntValueOutFunction { - + public MultiIntValueInIntValueOutRequireAllFunction(String name, TwoIntInIntOutLambda lambda, IntValue[] params) { super(name, lambda, params); } - + @Override public int getInt() { int value = params[0].getInt(); @@ -1577,11 +1577,11 @@ class MultiIntValueInIntValueOutRequireAllFunction extends MultiIntValueInIntVal } } class MultiIntValueInIntValueOutRequireOneFunction extends MultiIntValueInIntValueOutFunction { - + public MultiIntValueInIntValueOutRequireOneFunction(String name, TwoIntInIntOutLambda lambda, IntValue[] params) { super(name, lambda, params); } - + @Override public int getInt() { int i = -1; @@ -1609,7 +1609,7 @@ class LongValueInLongValueOutFunction extends AbstractLongValue { private final String name; private final String exprStr; private final ExpressionType funcType; - + public LongValueInLongValueOutFunction(String name, LongInLongOutLambda lambda, LongValue param) { this.name = name; this.lambda = lambda; @@ -1630,7 +1630,7 @@ class LongValueInLongValueOutFunction extends AbstractLongValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -1650,7 +1650,7 @@ class LongStreamInLongStreamOutFunction extends AbstractLongValueStream { private final String name; private final String exprStr; private final ExpressionType funcType; - + public LongStreamInLongStreamOutFunction(String name, LongInLongOutLambda lambda, LongValueStream param) { this.name = name; this.lambda = lambda; @@ -1663,7 +1663,7 @@ class LongStreamInLongStreamOutFunction extends AbstractLongValueStream { public void streamLongs(LongConsumer cons) { param.streamLongs(value -> cons.accept(lambda.apply(value))); } - + @Override public String getName() { return name; @@ -1683,7 +1683,7 @@ class LongStreamInLongValueOutFunction extends AbstractLongValue implements Long private final String name; private final String exprStr; private final ExpressionType funcType; - + public LongStreamInLongValueOutFunction(String name, TwoLongInLongOutLambda lambda, LongValueStream param) { this.name = name; this.lambda = lambda; @@ -1713,7 +1713,7 @@ class LongStreamInLongValueOutFunction extends AbstractLongValue implements Long value = lambda.apply(value, paramValue); } } - + @Override public String getName() { return name; @@ -1734,7 +1734,7 @@ class TwoLongValueInLongValueOutFunction extends AbstractLongValue { private final String name; private final String exprStr; private final ExpressionType funcType; - + public TwoLongValueInLongValueOutFunction(String name, TwoLongInLongOutLambda lambda, LongValue param1, LongValue param2) { this.name = name; this.lambda = lambda; @@ -1743,7 +1743,7 @@ class TwoLongValueInLongValueOutFunction extends AbstractLongValue { this.exprStr = AnalyticsValueStream.createExpressionString(name,param1,param2); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,param1,param2); } - + private boolean exists = false; @Override @@ -1756,7 +1756,7 @@ class TwoLongValueInLongValueOutFunction extends AbstractLongValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -1777,7 +1777,7 @@ class LongValueLongStreamInLongStreamOutFunction extends AbstractLongValueStream private final String name; private final String exprStr; private final ExpressionType funcType; - + public LongValueLongStreamInLongStreamOutFunction(String name, TwoLongInLongOutLambda lambda, LongValue param1, LongValueStream param2) { this.name = name; this.lambda = lambda; @@ -1815,7 +1815,7 @@ class LongStreamLongValueInLongStreamOutFunction extends AbstractLongValueStream private final String name; private final String exprStr; private final ExpressionType funcType; - + public LongStreamLongValueInLongStreamOutFunction(String name, TwoLongInLongOutLambda lambda, LongValueStream param1, LongValue param2) { this.name = name; this.lambda = lambda; @@ -1852,7 +1852,7 @@ abstract class MultiLongValueInLongValueOutFunction extends AbstractLongValue { private final String name; private final String exprStr; private final ExpressionType funcType; - + public MultiLongValueInLongValueOutFunction(String name, TwoLongInLongOutLambda lambda, LongValue[] params) { this.name = name; this.lambda = lambda; @@ -1860,14 +1860,14 @@ abstract class MultiLongValueInLongValueOutFunction extends AbstractLongValue { this.exprStr = AnalyticsValueStream.createExpressionString(name,params); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,params); } - + protected boolean exists = false; protected long temp; @Override public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -1882,11 +1882,11 @@ abstract class MultiLongValueInLongValueOutFunction extends AbstractLongValue { } } class MultiLongValueInLongValueOutRequireAllFunction extends MultiLongValueInLongValueOutFunction { - + public MultiLongValueInLongValueOutRequireAllFunction(String name, TwoLongInLongOutLambda lambda, LongValue[] params) { super(name, lambda, params); } - + @Override public long getLong() { long value = params[0].getLong(); @@ -1899,11 +1899,11 @@ class MultiLongValueInLongValueOutRequireAllFunction extends MultiLongValueInLon } } class MultiLongValueInLongValueOutRequireOneFunction extends MultiLongValueInLongValueOutFunction { - + public MultiLongValueInLongValueOutRequireOneFunction(String name, TwoLongInLongOutLambda lambda, LongValue[] params) { super(name, lambda, params); } - + @Override public long getLong() { int i = -1; @@ -1931,7 +1931,7 @@ class FloatValueInFloatValueOutFunction extends AbstractFloatValue { private final String name; private final String exprStr; private final ExpressionType funcType; - + public FloatValueInFloatValueOutFunction(String name, FloatInFloatOutLambda lambda, FloatValue param) { this.name = name; this.lambda = lambda; @@ -1952,7 +1952,7 @@ class FloatValueInFloatValueOutFunction extends AbstractFloatValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -1972,7 +1972,7 @@ class FloatStreamInFloatStreamOutFunction extends AbstractFloatValueStream { private final String name; private final String exprStr; private final ExpressionType funcType; - + public FloatStreamInFloatStreamOutFunction(String name, FloatInFloatOutLambda lambda, FloatValueStream param) { this.name = name; this.lambda = lambda; @@ -1985,7 +1985,7 @@ class FloatStreamInFloatStreamOutFunction extends AbstractFloatValueStream { public void streamFloats(FloatConsumer cons) { param.streamFloats(value -> cons.accept(lambda.apply(value))); } - + @Override public String getName() { return name; @@ -2005,7 +2005,7 @@ class FloatStreamInFloatValueOutFunction extends AbstractFloatValue implements F private final String name; private final String exprStr; private final ExpressionType funcType; - + public FloatStreamInFloatValueOutFunction(String name, TwoFloatInFloatOutLambda lambda, FloatValueStream param) { this.name = name; this.lambda = lambda; @@ -2035,7 +2035,7 @@ class FloatStreamInFloatValueOutFunction extends AbstractFloatValue implements F value = lambda.apply(value, paramValue); } } - + @Override public String getName() { return name; @@ -2056,7 +2056,7 @@ class TwoFloatValueInFloatValueOutFunction extends AbstractFloatValue { private final String name; private final String exprStr; private final ExpressionType funcType; - + public TwoFloatValueInFloatValueOutFunction(String name, TwoFloatInFloatOutLambda lambda, FloatValue param1, FloatValue param2) { this.name = name; this.lambda = lambda; @@ -2065,7 +2065,7 @@ class TwoFloatValueInFloatValueOutFunction extends AbstractFloatValue { this.exprStr = AnalyticsValueStream.createExpressionString(name,param1,param2); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,param1,param2); } - + private boolean exists = false; @Override @@ -2078,7 +2078,7 @@ class TwoFloatValueInFloatValueOutFunction extends AbstractFloatValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -2099,7 +2099,7 @@ class FloatValueFloatStreamInFloatStreamOutFunction extends AbstractFloatValueSt private final String name; private final String exprStr; private final ExpressionType funcType; - + public FloatValueFloatStreamInFloatStreamOutFunction(String name, TwoFloatInFloatOutLambda lambda, FloatValue param1, FloatValueStream param2) { this.name = name; this.lambda = lambda; @@ -2137,7 +2137,7 @@ class FloatStreamFloatValueInFloatStreamOutFunction extends AbstractFloatValueSt private final String name; private final String exprStr; private final ExpressionType funcType; - + public FloatStreamFloatValueInFloatStreamOutFunction(String name, TwoFloatInFloatOutLambda lambda, FloatValueStream param1, FloatValue param2) { this.name = name; this.lambda = lambda; @@ -2174,7 +2174,7 @@ abstract class MultiFloatValueInFloatValueOutFunction extends AbstractFloatValue private final String name; private final String exprStr; private final ExpressionType funcType; - + public MultiFloatValueInFloatValueOutFunction(String name, TwoFloatInFloatOutLambda lambda, FloatValue[] params) { this.name = name; this.lambda = lambda; @@ -2182,14 +2182,14 @@ abstract class MultiFloatValueInFloatValueOutFunction extends AbstractFloatValue this.exprStr = AnalyticsValueStream.createExpressionString(name,params); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,params); } - + protected boolean exists = false; protected float temp; @Override public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -2204,11 +2204,11 @@ abstract class MultiFloatValueInFloatValueOutFunction extends AbstractFloatValue } } class MultiFloatValueInFloatValueOutRequireAllFunction extends MultiFloatValueInFloatValueOutFunction { - + public MultiFloatValueInFloatValueOutRequireAllFunction(String name, TwoFloatInFloatOutLambda lambda, FloatValue[] params) { super(name, lambda, params); } - + @Override public float getFloat() { float value = params[0].getFloat(); @@ -2221,11 +2221,11 @@ class MultiFloatValueInFloatValueOutRequireAllFunction extends MultiFloatValueIn } } class MultiFloatValueInFloatValueOutRequireOneFunction extends MultiFloatValueInFloatValueOutFunction { - + public MultiFloatValueInFloatValueOutRequireOneFunction(String name, TwoFloatInFloatOutLambda lambda, FloatValue[] params) { super(name, lambda, params); } - + @Override public float getFloat() { int i = -1; @@ -2253,7 +2253,7 @@ class DoubleValueInDoubleValueOutFunction extends AbstractDoubleValue { private final String name; private final String exprStr; private final ExpressionType funcType; - + public DoubleValueInDoubleValueOutFunction(String name, DoubleInDoubleOutLambda lambda, DoubleValue param) { this.name = name; this.lambda = lambda; @@ -2274,7 +2274,7 @@ class DoubleValueInDoubleValueOutFunction extends AbstractDoubleValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -2294,7 +2294,7 @@ class DoubleStreamInDoubleStreamOutFunction extends AbstractDoubleValueStream { private final String name; private final String exprStr; private final ExpressionType funcType; - + public DoubleStreamInDoubleStreamOutFunction(String name, DoubleInDoubleOutLambda lambda, DoubleValueStream param) { this.name = name; this.lambda = lambda; @@ -2307,7 +2307,7 @@ class DoubleStreamInDoubleStreamOutFunction extends AbstractDoubleValueStream { public void streamDoubles(DoubleConsumer cons) { param.streamDoubles(value -> cons.accept(lambda.apply(value))); } - + @Override public String getName() { return name; @@ -2327,7 +2327,7 @@ class DoubleStreamInDoubleValueOutFunction extends AbstractDoubleValue implement private final String name; private final String exprStr; private final ExpressionType funcType; - + public DoubleStreamInDoubleValueOutFunction(String name, TwoDoubleInDoubleOutLambda lambda, DoubleValueStream param) { this.name = name; this.lambda = lambda; @@ -2357,7 +2357,7 @@ class DoubleStreamInDoubleValueOutFunction extends AbstractDoubleValue implement value = lambda.apply(value, paramValue); } } - + @Override public String getName() { return name; @@ -2378,7 +2378,7 @@ class TwoDoubleValueInDoubleValueOutFunction extends AbstractDoubleValue { private final String name; private final String exprStr; private final ExpressionType funcType; - + public TwoDoubleValueInDoubleValueOutFunction(String name, TwoDoubleInDoubleOutLambda lambda, DoubleValue param1, DoubleValue param2) { this.name = name; this.lambda = lambda; @@ -2387,7 +2387,7 @@ class TwoDoubleValueInDoubleValueOutFunction extends AbstractDoubleValue { this.exprStr = AnalyticsValueStream.createExpressionString(name,param1,param2); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,param1,param2); } - + private boolean exists = false; @Override @@ -2400,7 +2400,7 @@ class TwoDoubleValueInDoubleValueOutFunction extends AbstractDoubleValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -2421,7 +2421,7 @@ class DoubleValueDoubleStreamInDoubleStreamOutFunction extends AbstractDoubleVal private final String name; private final String exprStr; private final ExpressionType funcType; - + public DoubleValueDoubleStreamInDoubleStreamOutFunction(String name, TwoDoubleInDoubleOutLambda lambda, DoubleValue param1, DoubleValueStream param2) { this.name = name; this.lambda = lambda; @@ -2459,7 +2459,7 @@ class DoubleStreamDoubleValueInDoubleStreamOutFunction extends AbstractDoubleVal private final String name; private final String exprStr; private final ExpressionType funcType; - + public DoubleStreamDoubleValueInDoubleStreamOutFunction(String name, TwoDoubleInDoubleOutLambda lambda, DoubleValueStream param1, DoubleValue param2) { this.name = name; this.lambda = lambda; @@ -2496,7 +2496,7 @@ abstract class MultiDoubleValueInDoubleValueOutFunction extends AbstractDoubleVa private final String name; private final String exprStr; private final ExpressionType funcType; - + public MultiDoubleValueInDoubleValueOutFunction(String name, TwoDoubleInDoubleOutLambda lambda, DoubleValue[] params) { this.name = name; this.lambda = lambda; @@ -2504,14 +2504,14 @@ abstract class MultiDoubleValueInDoubleValueOutFunction extends AbstractDoubleVa this.exprStr = AnalyticsValueStream.createExpressionString(name,params); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,params); } - + protected boolean exists = false; protected double temp; @Override public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -2526,11 +2526,11 @@ abstract class MultiDoubleValueInDoubleValueOutFunction extends AbstractDoubleVa } } class MultiDoubleValueInDoubleValueOutRequireAllFunction extends MultiDoubleValueInDoubleValueOutFunction { - + public MultiDoubleValueInDoubleValueOutRequireAllFunction(String name, TwoDoubleInDoubleOutLambda lambda, DoubleValue[] params) { super(name, lambda, params); } - + @Override public double getDouble() { double value = params[0].getDouble(); @@ -2543,11 +2543,11 @@ class MultiDoubleValueInDoubleValueOutRequireAllFunction extends MultiDoubleValu } } class MultiDoubleValueInDoubleValueOutRequireOneFunction extends MultiDoubleValueInDoubleValueOutFunction { - + public MultiDoubleValueInDoubleValueOutRequireOneFunction(String name, TwoDoubleInDoubleOutLambda lambda, DoubleValue[] params) { super(name, lambda, params); } - + @Override public double getDouble() { int i = -1; @@ -2575,7 +2575,7 @@ class DateValueInDateValueOutFunction extends AbstractDateValue { private final String name; private final String exprStr; private final ExpressionType funcType; - + public DateValueInDateValueOutFunction(String name, LongInLongOutLambda lambda, DateValue param) { this.name = name; this.lambda = lambda; @@ -2596,7 +2596,7 @@ class DateValueInDateValueOutFunction extends AbstractDateValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -2616,7 +2616,7 @@ class DateStreamInDateStreamOutFunction extends AbstractDateValueStream { private final String name; private final String exprStr; private final ExpressionType funcType; - + public DateStreamInDateStreamOutFunction(String name, LongInLongOutLambda lambda, DateValueStream param) { this.name = name; this.lambda = lambda; @@ -2629,7 +2629,7 @@ class DateStreamInDateStreamOutFunction extends AbstractDateValueStream { public void streamLongs(LongConsumer cons) { param.streamLongs(value -> cons.accept(lambda.apply(value))); } - + @Override public String getName() { return name; @@ -2649,7 +2649,7 @@ class DateStreamInDateValueOutFunction extends AbstractDateValue implements Long private final String name; private final String exprStr; private final ExpressionType funcType; - + public DateStreamInDateValueOutFunction(String name, TwoLongInLongOutLambda lambda, DateValueStream param) { this.name = name; this.lambda = lambda; @@ -2679,7 +2679,7 @@ class DateStreamInDateValueOutFunction extends AbstractDateValue implements Long value = lambda.apply(value, paramValue); } } - + @Override public String getName() { return name; @@ -2700,7 +2700,7 @@ class TwoDateValueInDateValueOutFunction extends AbstractDateValue { private final String name; private final String exprStr; private final ExpressionType funcType; - + public TwoDateValueInDateValueOutFunction(String name, TwoLongInLongOutLambda lambda, DateValue param1, DateValue param2) { this.name = name; this.lambda = lambda; @@ -2709,7 +2709,7 @@ class TwoDateValueInDateValueOutFunction extends AbstractDateValue { this.exprStr = AnalyticsValueStream.createExpressionString(name,param1,param2); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,param1,param2); } - + private boolean exists = false; @Override @@ -2722,7 +2722,7 @@ class TwoDateValueInDateValueOutFunction extends AbstractDateValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -2743,7 +2743,7 @@ class DateValueDateStreamInDateStreamOutFunction extends AbstractDateValueStream private final String name; private final String exprStr; private final ExpressionType funcType; - + public DateValueDateStreamInDateStreamOutFunction(String name, TwoLongInLongOutLambda lambda, DateValue param1, DateValueStream param2) { this.name = name; this.lambda = lambda; @@ -2781,7 +2781,7 @@ class DateStreamDateValueInDateStreamOutFunction extends AbstractDateValueStream private final String name; private final String exprStr; private final ExpressionType funcType; - + public DateStreamDateValueInDateStreamOutFunction(String name, TwoLongInLongOutLambda lambda, DateValueStream param1, DateValue param2) { this.name = name; this.lambda = lambda; @@ -2818,7 +2818,7 @@ abstract class MultiDateValueInDateValueOutFunction extends AbstractDateValue { private final String name; private final String exprStr; private final ExpressionType funcType; - + public MultiDateValueInDateValueOutFunction(String name, TwoLongInLongOutLambda lambda, DateValue[] params) { this.name = name; this.lambda = lambda; @@ -2826,14 +2826,14 @@ abstract class MultiDateValueInDateValueOutFunction extends AbstractDateValue { this.exprStr = AnalyticsValueStream.createExpressionString(name,params); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,params); } - + protected boolean exists = false; protected long temp; @Override public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -2848,11 +2848,11 @@ abstract class MultiDateValueInDateValueOutFunction extends AbstractDateValue { } } class MultiDateValueInDateValueOutRequireAllFunction extends MultiDateValueInDateValueOutFunction { - + public MultiDateValueInDateValueOutRequireAllFunction(String name, TwoLongInLongOutLambda lambda, DateValue[] params) { super(name, lambda, params); } - + @Override public long getLong() { long value = params[0].getLong(); @@ -2865,11 +2865,11 @@ class MultiDateValueInDateValueOutRequireAllFunction extends MultiDateValueInDat } } class MultiDateValueInDateValueOutRequireOneFunction extends MultiDateValueInDateValueOutFunction { - + public MultiDateValueInDateValueOutRequireOneFunction(String name, TwoLongInLongOutLambda lambda, DateValue[] params) { super(name, lambda, params); } - + @Override public long getLong() { int i = -1; @@ -2897,7 +2897,7 @@ class StringValueInStringValueOutFunction extends AbstractStringValue { private final String name; private final String exprStr; private final ExpressionType funcType; - + public StringValueInStringValueOutFunction(String name, StringInStringOutLambda lambda, StringValue param) { this.name = name; this.lambda = lambda; @@ -2918,7 +2918,7 @@ class StringValueInStringValueOutFunction extends AbstractStringValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -2938,7 +2938,7 @@ class StringStreamInStringStreamOutFunction extends AbstractStringValueStream { private final String name; private final String exprStr; private final ExpressionType funcType; - + public StringStreamInStringStreamOutFunction(String name, StringInStringOutLambda lambda, StringValueStream param) { this.name = name; this.lambda = lambda; @@ -2951,7 +2951,7 @@ class StringStreamInStringStreamOutFunction extends AbstractStringValueStream { public void streamStrings(Consumer cons) { param.streamStrings(value -> cons.accept(lambda.apply(value))); } - + @Override public String getName() { return name; @@ -2971,7 +2971,7 @@ class StringStreamInStringValueOutFunction extends AbstractStringValue implement private final String name; private final String exprStr; private final ExpressionType funcType; - + public StringStreamInStringValueOutFunction(String name, TwoStringInStringOutLambda lambda, StringValueStream param) { this.name = name; this.lambda = lambda; @@ -3001,7 +3001,7 @@ class StringStreamInStringValueOutFunction extends AbstractStringValue implement value = lambda.apply(value, paramValue); } } - + @Override public String getName() { return name; @@ -3022,7 +3022,7 @@ class TwoStringValueInStringValueOutFunction extends AbstractStringValue { private final String name; private final String exprStr; private final ExpressionType funcType; - + public TwoStringValueInStringValueOutFunction(String name, TwoStringInStringOutLambda lambda, StringValue param1, StringValue param2) { this.name = name; this.lambda = lambda; @@ -3031,7 +3031,7 @@ class TwoStringValueInStringValueOutFunction extends AbstractStringValue { this.exprStr = AnalyticsValueStream.createExpressionString(name,param1,param2); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,param1,param2); } - + private boolean exists = false; @Override @@ -3044,7 +3044,7 @@ class TwoStringValueInStringValueOutFunction extends AbstractStringValue { public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -3065,7 +3065,7 @@ class StringValueStringStreamInStringStreamOutFunction extends AbstractStringVal private final String name; private final String exprStr; private final ExpressionType funcType; - + public StringValueStringStreamInStringStreamOutFunction(String name, TwoStringInStringOutLambda lambda, StringValue param1, StringValueStream param2) { this.name = name; this.lambda = lambda; @@ -3103,7 +3103,7 @@ class StringStreamStringValueInStringStreamOutFunction extends AbstractStringVal private final String name; private final String exprStr; private final ExpressionType funcType; - + public StringStreamStringValueInStringStreamOutFunction(String name, TwoStringInStringOutLambda lambda, StringValueStream param1, StringValue param2) { this.name = name; this.lambda = lambda; @@ -3140,7 +3140,7 @@ abstract class MultiStringValueInStringValueOutFunction extends AbstractStringVa private final String name; private final String exprStr; private final ExpressionType funcType; - + public MultiStringValueInStringValueOutFunction(String name, TwoStringInStringOutLambda lambda, StringValue[] params) { this.name = name; this.lambda = lambda; @@ -3148,14 +3148,14 @@ abstract class MultiStringValueInStringValueOutFunction extends AbstractStringVa this.exprStr = AnalyticsValueStream.createExpressionString(name,params); this.funcType = AnalyticsValueStream.determineMappingPhase(exprStr,params); } - + protected boolean exists = false; protected String temp = null; @Override public boolean exists() { return exists; } - + @Override public String getName() { return name; @@ -3170,11 +3170,11 @@ abstract class MultiStringValueInStringValueOutFunction extends AbstractStringVa } } class MultiStringValueInStringValueOutRequireAllFunction extends MultiStringValueInStringValueOutFunction { - + public MultiStringValueInStringValueOutRequireAllFunction(String name, TwoStringInStringOutLambda lambda, StringValue[] params) { super(name, lambda, params); } - + @Override public String getString() { String value = params[0].getString(); @@ -3192,11 +3192,11 @@ class MultiStringValueInStringValueOutRequireAllFunction extends MultiStringValu } } class MultiStringValueInStringValueOutRequireOneFunction extends MultiStringValueInStringValueOutFunction { - + public MultiStringValueInStringValueOutRequireOneFunction(String name, TwoStringInStringOutLambda lambda, StringValue[] params) { super(name, lambda, params); } - + @Override public String getString() { int i = -1; diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/LogFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/LogFunction.java index 313f2e212a7..7b42df106b2 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/LogFunction.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/LogFunction.java @@ -30,8 +30,8 @@ import org.apache.solr.common.SolrException.ErrorCode; *

  • If one numeric Value or ValueStream is passed in, a {@link DoubleValue} or {@link DoubleValueStream} * representing the natural logarithm is returned. *
  • If two numeric Values are passed in, a {@link DoubleValue} representing the logarithm of the first with the second as the base is returned. - *
  • If a numeric ValueStream and a numeric Value are passed in, a {@link DoubleValueStream} representing the logarithm of - * the Value with each of the values of the ValueStream for a document as the base is returned. + *
  • If a numeric ValueStream and a numeric Value are passed in, a {@link DoubleValueStream} representing the logarithm of + * the Value with each of the values of the ValueStream for a document as the base is returned. * (Or the other way, since the Value and ValueStream can be used in either order) * */ diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/LogicFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/LogicFunction.java index 93dea531d2f..6ad1fa6a570 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/LogicFunction.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/LogicFunction.java @@ -31,24 +31,24 @@ import org.apache.solr.common.SolrException.ErrorCode; *
      *
    • If a single {@link BooleanValueStream} is passed in, a {@link BooleanValue} representing the logical operation * on all of the values for each document is returned. - *
    • If a {@link BooleanValueStream} and a {@link BooleanValue} are passed in, a {@link BooleanValue} representing the logical operation on + *
    • If a {@link BooleanValueStream} and a {@link BooleanValue} are passed in, a {@link BooleanValue} representing the logical operation on * the {@link BooleanValue} and each of the values of the {@link BooleanValueStream} for a document is returned. * (Or the other way, since the Value and ValueStream can be used in either order) *
    • If multiple {@link BooleanValue}s are passed in, a {@link BooleanValue} representing the logical operation on all values is returned. *
    */ public class LogicFunction { - + private static BooleanValueStream createBitwiseFunction(String name, TwoBoolInBoolOutLambda comp, AnalyticsValueStream... params) { if (params.length == 0) { throw new SolrException(ErrorCode.BAD_REQUEST,"The "+name+" function requires parameters."); - } + } else if (params.length == 1) { if (params[0] instanceof BooleanValueStream) { return LambdaFunction.createBooleanLambdaFunction(name, comp, (BooleanValueStream)params[0]); } throw new SolrException(ErrorCode.BAD_REQUEST,"The "+name+" function requires boolean parameters. Incorrect param: "+params[0].getExpressionStr()); - } + } else if (params.length == 2) { AnalyticsValueStream param1 = params[0]; AnalyticsValueStream param2 = params[1]; @@ -67,7 +67,7 @@ public class LogicFunction { } return LambdaFunction.createBooleanLambdaFunction(name, comp, castedParams); }; - + /** * A mapping function for the logical operation AND. */ diff --git a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/MultFunction.java b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/MultFunction.java index 4a5b1736962..70500d80258 100644 --- a/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/MultFunction.java +++ b/solr/contrib/analytics/src/java/org/apache/solr/analytics/function/mapping/MultFunction.java @@ -29,7 +29,7 @@ import org.apache.solr.analytics.value.DoubleValueStream; * Uses: *