LUCENE-7419: Don't lookup PositionIncrementAttribute every time in TokenStream.end()

lucene/CHANGES.txt

@@ -91,6 +91,9 @@ Bug Fixes
 * SOLR-9413: Fix analysis/kuromoji's CSVUtil.quoteEscape logic, add TestCSVUtil test.
   (AppChecker, Christine Poerschke)
 
+* LUCENE-7419: Fix performance bug with TokenStream.end(), where it would lookup
+  PositionIncrementAttribute every time. (Mike McCandless, Robert Muir)
+
 Improvements
 
 * LUCENE-7323: Compound file writing now verifies the incoming

lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java

@@ -22,7 +22,6 @@ import java.io.Closeable;
 import java.lang.reflect.Modifier;
 
 import org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriter;
@@ -176,11 +175,7 @@ public abstract class TokenStream extends AttributeSource implements Closeable {
    * @throws IOException If an I/O error occurs
    */
   public void end() throws IOException {
-    clearAttributes(); // LUCENE-3849: don't consume dirty atts
-    PositionIncrementAttribute posIncAtt = getAttribute(PositionIncrementAttribute.class);
-    if (posIncAtt != null) {
-      posIncAtt.setPositionIncrement(0);
-    }
+    endAttributes(); // LUCENE-3849: don't consume dirty atts
   }
 
   /**

lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java

@@ -139,6 +139,17 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl
     type = DEFAULT_TYPE;
   }
   
+  /** Resets the attributes at end
+   */
+  @Override
+  public void end() {
+    super.end();
+    positionIncrement = 0;
+    positionLength = 1;
+    startOffset = endOffset = 0;
+    type = DEFAULT_TYPE;
+  }
+
   @Override
   public PackedTokenAttributeImpl clone() {
     return (PackedTokenAttributeImpl) super.clone();

lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java

@@ -46,6 +46,11 @@ public class PositionIncrementAttributeImpl extends AttributeImpl implements Pos
     this.positionIncrement = 1;
   }
   
+  @Override
+  public void end() {
+    this.positionIncrement = 0;
+  }
+
   @Override
   public boolean equals(Object other) {
     if (other == this) {

lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java

@@ -32,6 +32,17 @@ public abstract class AttributeImpl implements Cloneable, Attribute {
    */
   public abstract void clear();
   
+  /**
+   * Clears the values in this AttributeImpl and resets it to its value
+   * at the end of the field. If this implementation implements more than one Attribute interface
+   * it clears all.
+   * <p>
+   * The default implementation simply calls {@link #clear()}
+   */
+  public void end() {
+    clear();
+  }
+  
   /**
    * This method returns the current attribute values as a string in the following format
    * by calling the {@link #reflectWith(AttributeReflector)} method:

lucene/core/src/java/org/apache/lucene/util/AttributeSource.java

@@ -271,6 +271,16 @@ public class AttributeSource {
     }
   }
   
+  /**
+   * Resets all Attributes in this AttributeSource by calling
+   * {@link AttributeImpl#end()} on each Attribute implementation.
+   */
+  public final void endAttributes() {
+    for (State state = getCurrentState(); state != null; state = state.next) {
+      state.attribute.end();
+    }
+  }
+
   /**
    * Removes all attributes and their implementations from this AttributeSource.
    */