LUCENE-2764: Allow tests to use random codec per field

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1036543 13f79535-47bb-0310-9956-ffa450edef68

lucene/common-build.xml

@@ -65,6 +65,7 @@
   </condition>
   <property name="tests.multiplier" value="1" />
   <property name="tests.codec" value="random" />
+  <property name="tests.randomCodecPerField" value="true"/>
   <property name="tests.locale" value="random" />
   <property name="tests.timezone" value="random" />
   <property name="tests.directory" value="random" />

lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java

@@ -133,7 +133,6 @@ public class CreateIndexTask extends PerfTask {
 
     final String defaultCodec = config.get("default.codec", null);
     if (defaultCodec != null) {
-      CodecProvider.setDefaultCodec(defaultCodec);
       CodecProvider.getDefault().setDefaultFieldCodec(defaultCodec);
     }
 

lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java

@@ -39,7 +39,7 @@ import org.apache.lucene.index.codecs.standard.StandardCodec;
 public class CodecProvider {
   private SegmentInfosWriter infosWriter = new DefaultSegmentInfosWriter();
   private SegmentInfosReader infosReader = new DefaultSegmentInfosReader();
-  private String defaultFieldCodec = defaultCodec;
+  private String defaultFieldCodec = "Standard";
   private final Map<String, String> perFieldMap = new HashMap<String, String>();
 
   
@@ -47,7 +47,6 @@ public class CodecProvider {
 
   private final Set<String> knownExtensions = new HashSet<String>();
 
-  private static String defaultCodec = "Standard";
 
   public final static String[] CORE_CODECS = new String[] {"Standard", "Pulsing", "PreFlex", "SimpleText"};
 
@@ -102,15 +101,6 @@ public class CodecProvider {
   public static CodecProvider getDefault() {
     return defaultCodecs;
   }
-
-  /** Used for testing. @lucene.internal */
-  public synchronized static void setDefaultCodec(String s) {
-    defaultCodec = s;
-  }
-  /** Used for testing. @lucene.internal */
-  public synchronized static String getDefaultCodec() {
-    return defaultCodec;
-  }
   
   /**
    * Sets the {@link Codec} for a given field. Not that setting a fields code is
@@ -175,6 +165,5 @@ class DefaultCodecProvider extends CodecProvider {
     register(new PreFlexCodec());
     register(new PulsingCodec(1));
     register(new SimpleTextCodec());
-    setDefaultFieldCodec(CodecProvider.getDefaultCodec());
   }
 }

lucene/src/test/org/apache/lucene/index/RandomIndexWriter.java

@@ -24,7 +24,6 @@ import java.util.Random;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.Version;
@@ -84,7 +83,7 @@ public class RandomIndexWriter implements Closeable {
     flushAt = _TestUtil.nextInt(r, 10, 1000);
     if (LuceneTestCase.VERBOSE) {
       System.out.println("RIW config=" + w.getConfig());
-      System.out.println("codec default=" + CodecProvider.getDefaultCodec());
+      System.out.println("codec default=" + w.getConfig().getCodecProvider().getDefaultFieldCodec());
     }
   } 
 

lucene/src/test/org/apache/lucene/index/Test2BTerms.java

@@ -123,7 +123,7 @@ public class Test2BTerms extends LuceneTestCase {
   @Ignore("Takes ~4 hours to run on a fast machine!!  And requires that you don't use PreFlex codec.")
   public void test2BTerms() throws IOException {
 
-    if ("PreFlex".equals(CodecProvider.getDefaultCodec())) {
+    if ("PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())) {
       throw new RuntimeException("thist test cannot run with PreFlex codec");
     }
 

lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java

@@ -69,7 +69,7 @@ public class TestLazyProxSkipping extends LuceneTestCase {
         int numDocs = 500;
         
         Directory directory = new SeekCountingDirectory(new RAMDirectory());
-        IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, true, false)).setMaxBufferedDocs(10));
+        IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, true, false)).setMaxBufferedDocs(10));
         ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false);
         ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false);
         for (int i = 0; i < numDocs; i++) {
@@ -120,7 +120,7 @@ public class TestLazyProxSkipping extends LuceneTestCase {
     }
  
     public void testLazySkipping() throws IOException {
-        assumeFalse("This test cannot run with SimpleText codec", CodecProvider.getDefaultCodec().equals("SimpleText"));
+        assumeFalse("This test cannot run with SimpleText codec", getRandomFieldCodec(this.field).equals("SimpleText"));
         // test whether only the minimum amount of seeks()
         // are performed
         performTest(5);

lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java

@@ -27,7 +27,6 @@ import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.AttributeSource;
 import org.apache.lucene.util.BytesRef;
@@ -57,7 +56,7 @@ public class TestPrefixRandom extends LuceneTestCase {
 
     // we generate aweful prefixes: good for testing.
     // but for preflex codec, the test can be very slow, so use less iterations.
-    String codec = CodecProvider.getDefaultCodec();
+    final String codec = getRandomFieldCodec("field");
     int num = codec.equals("PreFlex") ? 200 * RANDOM_MULTIPLIER : 2000 * RANDOM_MULTIPLIER;
     for (int i = 0; i < num; i++) {
       field.setValue(_TestUtil.randomUnicodeString(random, 10));

lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java

@@ -58,7 +58,6 @@ public class TestRegexpRandom2 extends LuceneTestCase {
     RandomIndexWriter writer = new RandomIndexWriter(random, dir, 
         newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.KEYWORD, false))
         .setMaxBufferedDocs(_TestUtil.nextInt(random, 50, 1000)));
-    
     Document doc = new Document();
     Field field = newField("field", "", Field.Store.NO, Field.Index.NOT_ANALYZED);
     doc.add(field);
@@ -135,8 +134,7 @@ public class TestRegexpRandom2 extends LuceneTestCase {
   public void testRegexps() throws Exception {
     // we generate aweful regexps: good for testing.
     // but for preflex codec, the test can be very slow, so use less iterations.
-    String codec = CodecProvider.getDefaultCodec();
+    int num = getRandomFieldCodec("field").equals("PreFlex") ? 100 * RANDOM_MULTIPLIER : 1000 * RANDOM_MULTIPLIER;
-    int num = codec.equals("PreFlex") ? 100 * RANDOM_MULTIPLIER : 1000 * RANDOM_MULTIPLIER;
     for (int i = 0; i < num; i++) {
       String reg = AutomatonTestUtil.randomRegexp(random);
       assertSame(reg);

lucene/src/test/org/apache/lucene/util/LuceneTestCase.java

@@ -35,6 +35,8 @@ import org.apache.lucene.index.codecs.mocksep.MockSepCodec;
 import org.apache.lucene.index.codecs.preflex.PreFlexCodec;
 import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec;
 import org.apache.lucene.index.codecs.pulsing.PulsingCodec;
+import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec;
+import org.apache.lucene.index.codecs.standard.StandardCodec;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.FieldCache;
 import org.apache.lucene.search.FieldCache.CacheEntry;
@@ -73,6 +75,7 @@ import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.IdentityHashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -146,6 +149,8 @@ public abstract class LuceneTestCase extends Assert {
   // tests)
   /** Gets the codec to run tests with. */
   static final String TEST_CODEC = System.getProperty("tests.codec", "random");
+  /** Get if a random codec per field should be used */ // only use random per field if no explicit codec is set
+  static final boolean TEST_RANDOM_CODEC_PER_FIELD = "random".equals(TEST_CODEC) && Boolean.parseBoolean(System.getProperty("tests.randomCodecPerField", "true"));
   /** Gets the locale to run tests with */
   static final String TEST_LOCALE = System.getProperty("tests.locale", "random");
   /** Gets the timezone to run tests with */
@@ -215,7 +220,7 @@ public abstract class LuceneTestCase extends Assert {
   static Codec installTestCodecs() {
     final CodecProvider cp = CodecProvider.getDefault();
 
-    savedDefaultCodec = CodecProvider.getDefaultCodec();
+    savedDefaultCodec = cp.getDefaultFieldCodec();
     String codec = TEST_CODEC;
 
     final boolean codecHasParam;
@@ -235,7 +240,6 @@ public abstract class LuceneTestCase extends Assert {
       }
     }
 
-    CodecProvider.setDefaultCodec(codec);
     cp.setDefaultFieldCodec(codec);
 
     if (codec.equals("PreFlex")) {
@@ -268,7 +272,6 @@ public abstract class LuceneTestCase extends Assert {
     cp.unregister(cp.lookup("MockFixedIntBlock"));
     cp.unregister(cp.lookup("MockVariableIntBlock"));
     swapCodec(new PulsingCodec(1));
-    CodecProvider.setDefaultCodec(savedDefaultCodec);
     cp.setDefaultFieldCodec(savedDefaultCodec);
 
   }
@@ -322,6 +325,7 @@ public abstract class LuceneTestCase extends Assert {
     timeZone = TEST_TIMEZONE.equals("random") ? randomTimeZone(random) : TimeZone.getTimeZone(TEST_TIMEZONE);
     TimeZone.setDefault(timeZone);
     testsFailed = false;
+    randomCodecProvider = new RandomCodecProvider(random);
   }
   
   @AfterClass
@@ -342,11 +346,11 @@ public abstract class LuceneTestCase extends Assert {
       }
     stores = null;
     // if tests failed, report some information back
-    if (testsFailed)
+    if (testsFailed) {
       System.out.println("NOTE: test params are: codec=" + codec + 
         ", locale=" + locale + 
-        ", timezone=" + (timeZone == null ? "(null)" : timeZone.getID()));
+        ", timezone=" + (timeZone == null ? "(null)" : timeZone.getID()) + 
-    if (testsFailed) {
+       (TEST_RANDOM_CODEC_PER_FIELD?", "+randomCodecProvider.toString():""));
       System.err.println("NOTE: all tests run in this JVM:");
       System.err.println(Arrays.toString(testClassesRun.toArray()));
     }
@@ -617,8 +621,9 @@ public abstract class LuceneTestCase extends Assert {
     return newIndexWriterConfig(random, v, a);
   }
   
+  /** create a new index writer config with random defaults */
   public static IndexWriterConfig newIndexWriterConfig(Random r, Version v, Analyzer a) {
-    IndexWriterConfig c = new IndexWriterConfig(v, a);
+    final IndexWriterConfig c = new IndexWriterConfig(v, a);
     if (r.nextBoolean()) {
       c.setMergePolicy(new LogDocMergePolicy());
     }
@@ -642,7 +647,9 @@ public abstract class LuceneTestCase extends Assert {
       logmp.setCalibrateSizeByDeletes(r.nextBoolean());
       logmp.setMergeFactor(_TestUtil.nextInt(r, 2, 20));
     }
-    
+    if (TEST_RANDOM_CODEC_PER_FIELD) {
+      c.setCodecProvider(randomCodecProvider);
+    }
     c.setReaderPooling(r.nextBoolean());
     c.setReaderTermsIndexDivisor(_TestUtil.nextInt(r, 1, 4));
     return c;
@@ -844,6 +851,8 @@ public abstract class LuceneTestCase extends Assert {
   // seed for individual test methods, changed in @before
   private long seed;
   
+  protected static CodecProvider randomCodecProvider;
+  
   private static final Random seedRand = new Random();
   protected static final Random random = new Random();
 
@@ -944,6 +953,61 @@ public abstract class LuceneTestCase extends Assert {
     }
   }
   
+  /** Returns the codec for the given field uses in this testcase */
+  public static String getRandomFieldCodec(String field) {
+    final CodecProvider provider = TEST_RANDOM_CODEC_PER_FIELD ? randomCodecProvider : CodecProvider.getDefault();
+    return provider.getFieldCodec(field);
+  }
+  
+  static class RandomCodecProvider extends CodecProvider {
+
+    private final Codec[] codecs;
+    private final Map<String, String> perFieldMap = new HashMap<String, String>();
+    private final Random random;
+
+    RandomCodecProvider(Random random) {
+      if (random.nextInt(5) == 0) {
+        /*
+         * We randomly swap in a exclusive PreFlexCodec to provide random test coverage
+         * for preFlex indexes. In realistic a PreFlex-Environment newer codecs don't occur.    
+         */
+        this.codecs = new Codec[] { new PreFlexRWCodec() };
+      } else {
+        this.codecs = new Codec[] { new StandardCodec(), new SimpleTextCodec(),
+            new MockSepCodec(), new PulsingCodec(1 + random.nextInt(10)),
+            new MockVariableIntBlockCodec(1 + random.nextInt(10)),
+            new MockFixedIntBlockCodec(1 + random.nextInt(10)), };
+        register(new PreFlexCodec()); // register this for read support
+      }
+      for (int i = 0; i < codecs.length; i++) {
+        register(codecs[i]);
+      }
+
+      this.random = random;
+    }
+
+    @Override
+    public synchronized String getFieldCodec(String name) {
+      if (!perFieldMap.containsKey(name)) { // select a codec at random
+        setFieldCodec(name, codecs[random.nextInt(codecs.length)].name);
+      }
+      return super.getFieldCodec(name);
+    }
+
+    @Override
+    public synchronized void setFieldCodec(String field, String codec) {
+      if (!perFieldMap.containsKey(field)) {
+        perFieldMap.put(field, codec);
+      }
+      super.setFieldCodec(field, codec);
+    }
+
+    @Override
+    public String toString() {
+      return "RandomCodecProvider [perFieldMap=" + perFieldMap + "]";
+    }
+  }
+  
   @Ignore("just a hack")
   public final void alwaysIgnoredTestMethod() {}
 }