LUCENE-2682: improve Test2BTerms to not generate pathological (sequential) terms for the BytesRefHash; they are generated fully randomly now

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1005356 13f79535-47bb-0310-9956-ffa450edef68

lucene/src/java/org/apache/lucene/index/DocumentsWriter.java

@@ -737,10 +737,12 @@ final class DocumentsWriter {
    * been acquired. */
   synchronized DocumentsWriterThreadState getThreadState(Document doc, Term delTerm) throws IOException {
 
+    final Thread currentThread = Thread.currentThread();
+
     // First, find a thread state.  If this thread already
     // has affinity to a specific ThreadState, use that one
     // again.
-    DocumentsWriterThreadState state = threadBindings.get(Thread.currentThread());
+    DocumentsWriterThreadState state = threadBindings.get(currentThread);
     if (state == null) {
 
       // First time this thread has called us since last
@@ -762,7 +764,7 @@ final class DocumentsWriter {
         state = newArray[threadStates.length] = new DocumentsWriterThreadState(this);
         threadStates = newArray;
       }
-      threadBindings.put(Thread.currentThread(), state);
+      threadBindings.put(currentThread, state);
     }
 
     // Next, wait until my thread state is idle (in case

lucene/src/test/org/apache/lucene/index/Test2BTerms.java

@@ -34,43 +34,35 @@ import org.junit.Ignore;
 // disk (but, should run successfully).  Best to run w/
 // -Dtests.codec=Standard, and w/ plenty of RAM, eg:
 //
-//   ant compile-core compile-test
+//   ant compile-test
 //
-//  java -server -Xmx2g -Xms2g -d64 -cp .:lib/junit-4.7.jar:./build/classes/test:./build/classes/java:./build/classes/demo -Dlucene.version=4.0-dev -Dtests.codec=Standard -DtempDir=build -ea org.junit.runner.JUnitCore org.apache.lucene.index.Test2BTerms
+//   java -server -Xmx2g -Xms2g -d64 -cp .:lib/junit-4.7.jar:./build/classes/test:./build/classes/java -Dlucene.version=4.0-dev -Dtests.directory=SimpleFSDirectory -Dtests.codec=Standard -DtempDir=build -ea org.junit.runner.JUnitCore org.apache.lucene.index.Test2BTerms
 //
 
 public class Test2BTerms extends LuceneTestCase {
 
-  private final static BytesRef bytes = new BytesRef(20);
+  private final static int TOKEN_LEN = 10;
+
+  private final static BytesRef bytes = new BytesRef(TOKEN_LEN);
 
   private static final class MyTokenStream extends TokenStream {
 
     private final int tokensPerDoc;
     private int tokenCount;
+    private int byteUpto;
 
     public MyTokenStream(int tokensPerDoc) {
       super(new MyAttributeFactory(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY));
       this.tokensPerDoc = tokensPerDoc;
       addAttribute(TermToBytesRefAttribute.class);
+      bytes.length = TOKEN_LEN;
     }
     
     public boolean incrementToken() {
       if (tokenCount >= tokensPerDoc) {
         return false;
       }
-      final byte[] bs = bytes.bytes;
+      random.nextBytes(bytes.bytes);
-      for(int i=bytes.length-1;i>=0;i--) {
-        int b = bs[i]&0xff;
-        if (b == 0xff) {
-          bs[i] = 0;
-        } else {
-          bs[i] = (byte) (++b);
-          tokenCount++;
-          return true;
-        }
-      }
-      bytes.length++;
-      bs[0] = 1;
       tokenCount++;
       return true;
     }
@@ -141,7 +133,7 @@ public class Test2BTerms extends LuceneTestCase {
 
     Directory dir = FSDirectory.open(_TestUtil.getTempDir("2BTerms"));
     IndexWriter w = new IndexWriter(dir,
-                                    newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())
+                                    new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())
                                                   .setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
                                                 .setRAMBufferSizeMB(256.0).setMergeScheduler(new ConcurrentMergeScheduler()));
     ((LogMergePolicy) w.getConfig().getMergePolicy()).setUseCompoundFile(false);
@@ -156,14 +148,19 @@ public class Test2BTerms extends LuceneTestCase {
     //w.setInfoStream(System.out);
     final int numDocs = (int) (TERM_COUNT/TERMS_PER_DOC);
     for(int i=0;i<numDocs;i++) {
+      final long t0 = System.currentTimeMillis();
       w.addDocument(doc);
-      System.out.println(i + " of " + numDocs);
+      System.out.println(i + " of " + numDocs + " " + (System.currentTimeMillis()-t0) + " msec");
     }
     System.out.println("now optimize...");
     w.optimize();
     w.close();
 
-    _TestUtil.checkIndex(dir);
+    System.out.println("now CheckIndex...");
+    CheckIndex.Status status = _TestUtil.checkIndex(dir);
+    final long tc = status.segmentInfos.get(0).termIndexStatus.termCount;
+    assertTrue("count " + tc + " is not > " + Integer.MAX_VALUE, tc > Integer.MAX_VALUE);
+
     dir.close();
   }
 }

lucene/src/test/org/apache/lucene/index/TestIndexWriter.java

@@ -77,7 +77,6 @@ import org.apache.lucene.util._TestUtil;
 import org.apache.lucene.util.ThreadInterruptedException;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Bits;
-import org.junit.Assume;
 
 public class TestIndexWriter extends LuceneTestCase {
 
@@ -3532,7 +3531,7 @@ public class TestIndexWriter extends LuceneTestCase {
     assertEquals(0, tps.nextPosition());
     w.close();
 
-    assertTrue(_TestUtil.checkIndex(dir));
+    _TestUtil.checkIndex(dir);
     s.close();
     dir.close();
   }
@@ -5275,7 +5274,6 @@ public class TestIndexWriter extends LuceneTestCase {
   // LUCENE-2593
   public void testCorruptionAfterDiskFullDuringMerge() throws IOException {
     MockDirectoryWrapper dir = newDirectory();
-    final Random rand = random;
     //IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setReaderPooling(true));
     IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergeScheduler(new SerialMergeScheduler()).setReaderPooling(true));
 

lucene/src/test/org/apache/lucene/util/_TestUtil.java

@@ -68,7 +68,7 @@ public class _TestUtil {
   /** This runs the CheckIndex tool on the index in.  If any
    *  issues are hit, a RuntimeException is thrown; else,
    *  true is returned. */
-  public static boolean checkIndex(Directory dir) throws IOException {
+  public static CheckIndex.Status checkIndex(Directory dir) throws IOException {
     ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
 
     CheckIndex checker = new CheckIndex(dir);
@@ -78,8 +78,9 @@ public class _TestUtil {
       System.out.println("CheckIndex failed");
       System.out.println(bos.toString());
       throw new RuntimeException("CheckIndex failed");
-    } else
+    } else {
-      return true;
+      return indexStatus;
+    }
   }
 
   /** start and end are BOTH inclusive */