From b3f73db5378655f9bddc90469b75862397a04a20 Mon Sep 17 00:00:00 2001
From: Karl-Johan Wettin <kalle@apache.org>
Date: Mon, 5 Oct 2009 16:01:17 +0000
Subject: [PATCH] LUCENE-1939: IndexOutOfBoundsException at
 ShingleMatrixFilter's Iterator#hasNext method on exhausted streams.

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@821888 13f79535-47bb-0310-9956-ffa450edef68
---
 contrib/CHANGES.txt                           |  4 +++
 .../analysis/shingle/ShingleMatrixFilter.java |  3 +-
 .../shingle/TestShingleMatrixFilter.java      | 31 +++++++++++++------
 3 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/contrib/CHANGES.txt b/contrib/CHANGES.txt
index 9108ad9bcc9..57e3a96dc6f 100644
--- a/contrib/CHANGES.txt
+++ b/contrib/CHANGES.txt
@@ -22,6 +22,10 @@ Bug fixes
    distance filter created for radius search in contrib/spatial.
    (Bill Bell via Mike McCandless)
 
+ * LUCENE-1939: IndexOutOfBoundsException at ShingleMatrixFilter's
+   Iterator#hasNext method on exhausted streams.
+   (Patrick Jungermann via Karl Wettin)
+
 New features
 
  * LUCENE-1924: Added BalancedSegmentMergePolicy to contrib/misc,
diff --git a/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java b/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java
index 3cd126f677c..75ebbb979d4 100644
--- a/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java
+++ b/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java
@@ -831,7 +831,8 @@ public class ShingleMatrixFilter extends TokenStream {
 
         public boolean hasNext() {
           int s = columnRowCounters.length;
-          return s != 0 && columnRowCounters[s - 1] < (columns.get(s - 1)).getRows().size();
+          int n = columns.size();
+          return s != 0 && n >= s && columnRowCounters[s - 1] < (columns.get(s - 1)).getRows().size();
         }
 
         public Column.Row[] next() {
diff --git a/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java b/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java
index bba73f77b00..64fc5ed1dfc 100644
--- a/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java
+++ b/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java
@@ -25,10 +25,7 @@ import java.util.LinkedList;
 import java.util.HashSet;
 import java.util.Arrays;
 
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.CachingTokenFilter;
-import org.apache.lucene.analysis.Token;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.*;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
 import org.apache.lucene.analysis.miscellaneous.EmptyTokenStream;
 import org.apache.lucene.analysis.miscellaneous.PrefixAndSuffixAwareTokenFilter;
@@ -42,11 +39,25 @@ public class TestShingleMatrixFilter extends BaseTokenStreamTestCase {
 
   public TestShingleMatrixFilter(String name) {
     // use this ctor, because SingleTokenTokenStream only uses next(Token), so exclude it
-    super(name, new HashSet(Arrays.asList(new String[]{
-      "testBehavingAsShingleFilter", "testMatrix"
-    })));
+    super(name, new HashSet<String>(Arrays.asList("testBehavingAsShingleFilter", "testMatrix", "testIterator")));
   }
-    
+
+  public void testIterator() throws IOException {
+
+    WhitespaceTokenizer wst = new WhitespaceTokenizer(new StringReader("one two three four five"));
+    ShingleMatrixFilter smf = new ShingleMatrixFilter(wst, 2, 2, '_', false, new ShingleMatrixFilter.OneDimensionalNonWeightedTokenSettingsCodec());
+
+    int i;
+    for(i=0; smf.incrementToken(); i++);
+    assertEquals(4, i);
+
+    // call next once more. this should return false again rather than throwing an exception (LUCENE-1939)
+    assertFalse(smf.incrementToken());
+
+    System.currentTimeMillis();
+
+  }
+
   public void testBehavingAsShingleFilter() throws IOException {
 
     ShingleMatrixFilter.defaultSettingsCodec = null;
@@ -466,8 +477,8 @@ public class TestShingleMatrixFilter extends BaseTokenStreamTestCase {
     TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
     PositionIncrementAttribute posIncrAtt = ts.addAttribute(PositionIncrementAttribute.class);
     PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class);
-    
-    assertTrue(ts.incrementToken());    
+
+    assertTrue(ts.incrementToken());
     assertEquals(text, termAtt.term());
     assertEquals(positionIncrement, posIncrAtt.getPositionIncrement());
     assertEquals(boost, payloadAtt.getPayload() == null ? 1f : PayloadHelper.decodeFloat(payloadAtt.getPayload().getData()), 0);