From 42ae21cb9ae36dde4d5ffda16e06976b369b95e4 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Sat, 13 Feb 2016 18:02:57 +0100 Subject: [PATCH] LUCENE-7027: Fixed NumericTermAttribute to not throw IllegalArgumentException after NumericTokenStream was exhausted --- lucene/CHANGES.txt | 4 ++ .../analysis/LegacyNumericTokenStream.java | 4 +- .../analysis/TestNumericTokenStream.java | 56 ++++++++++++++++++- 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 5d606d0ed02..aa603ba0d84 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -241,6 +241,10 @@ Bug Fixes * LUCENE-7002: Fixed MultiCollector to not throw a NPE if setScorer is called after one of the sub collectors is done collecting. (John Wang, Adrien Grand) +* LUCENE-7027: Fixed NumericTermAttribute to not throw IllegalArgumentException + after NumericTokenStream was exhausted. (Uwe Schindler, Lee Hinman, + Mike McCandless) + Other * LUCENE-6924: Upgrade randomizedtesting to 2.3.2. (Dawid Weiss) diff --git a/lucene/core/src/java/org/apache/lucene/analysis/LegacyNumericTokenStream.java b/lucene/core/src/java/org/apache/lucene/analysis/LegacyNumericTokenStream.java index 9bab26fd616..b382b10d859 100644 --- a/lucene/core/src/java/org/apache/lucene/analysis/LegacyNumericTokenStream.java +++ b/lucene/core/src/java/org/apache/lucene/analysis/LegacyNumericTokenStream.java @@ -157,7 +157,9 @@ public final class LegacyNumericTokenStream extends TokenStream { @Override public BytesRef getBytesRef() { assert valueSize == 64 || valueSize == 32; - if (valueSize == 64) { + if (shift >= valueSize) { + bytes.clear(); + } else if (valueSize == 64) { LegacyNumericUtils.longToPrefixCoded(value, shift, bytes); } else { LegacyNumericUtils.intToPrefixCoded((int) value, shift, bytes); diff --git a/lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java b/lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java index eb77bf4ae02..837b38abb46 100644 --- a/lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java +++ b/lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java @@ -17,6 +17,7 @@ package org.apache.lucene.analysis; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LegacyNumericUtils; import org.apache.lucene.analysis.LegacyNumericTokenStream.LegacyNumericTermAttributeImpl; import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; @@ -25,10 +26,11 @@ import org.apache.lucene.analysis.tokenattributes.TypeAttribute; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl; +@Deprecated public class TestNumericTokenStream extends BaseTokenStreamTestCase { - static final long lvalue = 4573245871874382L; - static final int ivalue = 123456; + final long lvalue = random().nextLong(); + final int ivalue = random().nextInt(); public void testLongStream() throws Exception { @SuppressWarnings("resource") @@ -116,13 +118,61 @@ public class TestNumericTokenStream extends BaseTokenStreamTestCase { stream.close(); } + /** LUCENE-7027 */ + public void testCaptureStateAfterExhausted() throws Exception { + // default precstep + try (LegacyNumericTokenStream stream=new LegacyNumericTokenStream()) { + // int + stream.setIntValue(ivalue); + stream.reset(); + while (stream.incrementToken()); + stream.captureState(); + stream.end(); + stream.captureState(); + // long + stream.setLongValue(lvalue); + stream.reset(); + while (stream.incrementToken()); + stream.captureState(); + stream.end(); + stream.captureState(); + } + // huge precstep + try (LegacyNumericTokenStream stream=new LegacyNumericTokenStream(Integer.MAX_VALUE)) { + // int + stream.setIntValue(ivalue); + stream.reset(); + while (stream.incrementToken()); + stream.captureState(); + stream.end(); + stream.captureState(); + // long + stream.setLongValue(lvalue); + stream.reset(); + while (stream.incrementToken()); + stream.captureState(); + stream.end(); + stream.captureState(); + } + } + public void testAttributeClone() throws Exception { LegacyNumericTermAttributeImpl att = new LegacyNumericTermAttributeImpl(); - att.init(1234L, 64, 8, 0); // set some value, to make getBytesRef() work + att.init(lvalue, 64, 8, 0); // set some value, to make getBytesRef() work LegacyNumericTermAttributeImpl copy = TestCharTermAttributeImpl.assertCloneIsEqual(att); assertNotSame(att.getBytesRef(), copy.getBytesRef()); LegacyNumericTermAttributeImpl copy2 = TestCharTermAttributeImpl.assertCopyIsEqual(att); assertNotSame(att.getBytesRef(), copy2.getBytesRef()); + + // LUCENE-7027 test + att.init(lvalue, 64, 8, 64); // Exhausted TokenStream -> should return empty BytesRef + assertEquals(new BytesRef(), att.getBytesRef()); + copy = TestCharTermAttributeImpl.assertCloneIsEqual(att); + assertEquals(new BytesRef(), copy.getBytesRef()); + assertNotSame(att.getBytesRef(), copy.getBytesRef()); + copy2 = TestCharTermAttributeImpl.assertCopyIsEqual(att); + assertEquals(new BytesRef(), copy2.getBytesRef()); + assertNotSame(att.getBytesRef(), copy2.getBytesRef()); } }