HBASE-19696 Filter returning INCLUDE_AND_NEXT_COL doesn't skip remaining versions when scan has explicit columns (Ankit Singhal)

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ColumnTracker.java

@@ -126,4 +126,13 @@ public interface ColumnTracker extends ShipperListener {
    * @return <code>true</code> to early out based on timestamp.
    */
   boolean isDone(long timestamp);
+
+  /**
+   * This method is used to inform the column tracker that we are done with this column. We may get
+   * this information from external filters or timestamp range and we then need to indicate this
+   * information to tracker. It is currently implemented for ExplicitColumnTracker.
+   * @param cell
+   */
+  default void doneWithColumn(Cell cell) {
+  }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/ExplicitColumnTracker.java

@@ -207,12 +207,7 @@ public class ExplicitColumnTracker implements ColumnTracker {
     return timestamp < oldestStamp;
   }
 
-  /**
+  @Override
-   * This method is used to inform the column tracker that we are done with this column. We may get
-   * this information from external filters or timestamp range and we then need to indicate this
-   * information to tracker. It is required only in case of ExplicitColumnTracker.
-   * @param cell
-   */
   public void doneWithColumn(Cell cell) {
     while (this.column != null) {
       int compare = CellUtil.compareQualifiers(cell, column.getBuffer(), column.getOffset(),

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/querymatcher/UserScanQueryMatcher.java

@@ -212,6 +212,9 @@ public abstract class UserScanQueryMatcher extends ScanQueryMatcher {
       case INCLUDE_AND_NEXT_COL:
         if (matchCode == MatchCode.INCLUDE) {
           matchCode = MatchCode.INCLUDE_AND_SEEK_NEXT_COL;
+        // Update column tracker to next column, As we use the column hint from the tracker to seek
+        // to next cell
+          columns.doneWithColumn(cell);
         }
         break;
       case INCLUDE_AND_SEEK_NEXT_ROW:

hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java

@@ -142,7 +142,8 @@ public class TestFilter {
   @Before
   public void setUp() throws Exception {
     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestFilter"));
-    htd.addFamily(new HColumnDescriptor(FAMILIES[0]));
+    HColumnDescriptor family0 = new HColumnDescriptor(FAMILIES[0]).setVersions(100, 100);
+    htd.addFamily(family0);
     htd.addFamily(new HColumnDescriptor(FAMILIES[1]));
     htd.addFamily(new HColumnDescriptor(FAMILIES_1[0]));
     htd.addFamily(new HColumnDescriptor(FAMILIES_1[1]));
@@ -1860,6 +1861,65 @@ public class TestFilter {
     this.verifyScanFull(s, expectedKVs3);
   }
 
+  @Test
+  public void testLatestVersionFilterWithExplicitColumn() throws Exception {
+    // Add multiple versions
+    Put p = new Put(ROWS_ONE[0]);
+    p.setDurability(Durability.SKIP_WAL);
+    p.addColumn(FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]);
+    this.region.put(p);
+    p = new Put(ROWS_ONE[0]);
+    p.setDurability(Durability.SKIP_WAL);
+    p.addColumn(FAMILIES[0], QUALIFIERS_ONE[0], VALUES[1]);
+    this.region.put(p);
+    this.region.flush(true);
+    Scan s = new Scan();
+    s.setFilter(new FilterBase() {
+      @Override
+      public ReturnCode filterCell(Cell c) throws IOException {
+        return ReturnCode.INCLUDE_AND_NEXT_COL;
+      }
+    });
+    s.readVersions(100);
+    s.addColumn(FAMILIES[0], QUALIFIERS_ONE[0]);
+    s.addColumn(FAMILIES[0], QUALIFIERS_ONE[1]);
+    s.addColumn(FAMILIES[0], QUALIFIERS_ONE[2]);
+    s.addColumn(FAMILIES[0], QUALIFIERS_ONE[3]);
+    s.addColumn(FAMILIES[0], QUALIFIERS_TWO[0]);
+    s.addColumn(FAMILIES[0], QUALIFIERS_TWO[1]);
+    s.addColumn(FAMILIES[0], QUALIFIERS_TWO[2]);
+    s.addColumn(FAMILIES[0], QUALIFIERS_TWO[3]);
+    KeyValue[] kvs = {
+      // testRowOne-0
+      new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[1]),
+      new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
+      new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
+
+      // testRowOne-2
+      new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
+      new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
+      new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
+
+      // testRowOne-3
+      new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
+      new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
+      new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
+      // testRowTwo-0
+      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
+      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
+      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
+      // testRowTwo-2
+      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
+      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
+      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
+      // testRowTwo-3
+      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
+      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
+      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), };
+    verifyScanFull(s, kvs);
+
+  }
+
   @Test
   public void testColumnPaginationFilter() throws Exception {
     // Test that the filter skips multiple column versions.