LUCENE-3969: add failing test case for MappingCharFilter wrong final offset

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene3969@1311761 13f79535-47bb-0310-9956-ffa450edef68

modules/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java

@@ -190,4 +190,28 @@ public class TestMappingCharFilter extends BaseTokenStreamTestCase {
     int numRounds = RANDOM_MULTIPLIER * 10000;
     checkRandomData(random, analyzer, numRounds);
   }
+  
+  // nocommit: wrong final offset, fix this!
+  public void testFinalOffsetSpecialCase() throws Exception {  
+    final NormalizeCharMap map = new NormalizeCharMap();
+    map.add("t", "");
+    // even though this below rule has no effect, the test passes if you remove it!!
+    map.add("tmakdbl", "c");
+    
+    Analyzer analyzer = new Analyzer() {
+      @Override
+      protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+        Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+        return new TokenStreamComponents(tokenizer, tokenizer);
+      }
+
+      @Override
+      protected Reader initReader(Reader reader) {
+        return new MappingCharFilter(map, CharReader.get(reader));
+      }
+    };
+    
+    String text = "gzw f quaxot";
+    checkAnalysisConsistency(random, analyzer, false, text);
+  }
 }

modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java

@@ -389,11 +389,14 @@ public class TestRandomChains extends BaseTokenStreamTestCase {
         // we can't add duplicate keys, or NormalizeCharMap gets angry
         Set<String> keys = new HashSet<String>();
         int num = random.nextInt(5);
+        //System.out.println("NormalizeCharMap=");
         for (int i = 0; i < num; i++) {
           String key = _TestUtil.randomSimpleString(random);
           if (!keys.contains(key)) {
-            map.add(key,_TestUtil.randomSimpleString(random));
+            String value = _TestUtil.randomSimpleString(random);
+            map.add(key, value);
             keys.add(key);
+            //System.out.println("mapping: '" + key + "' => '" + value + "'");
           }
         }
         return map;