Branch for LUCENE-5622

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/LUCENE-5622@1589645 13f79535-47bb-0310-9956-ffa450edef68

lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestBugInSomething.java

@@ -219,7 +219,7 @@ public class TestBugInSomething extends BaseTokenStreamTestCase {
     @Override
     public boolean incrementToken() throws IOException {
       if (input.incrementToken()) {
-        System.out.println(input.getClass().getSimpleName() + "->" + this.reflectAsString(false));
+        if (VERBOSE) System.out.println(input.getClass().getSimpleName() + "->" + this.reflectAsString(false));
         return true;
       } else {
         return false;
@@ -229,19 +229,19 @@ public class TestBugInSomething extends BaseTokenStreamTestCase {
     @Override
     public void end() throws IOException {
       super.end();
-      System.out.println(input.getClass().getSimpleName() + ".end()");
+      if (VERBOSE) System.out.println(input.getClass().getSimpleName() + ".end()");
     }
 
     @Override
     public void close() throws IOException {
       super.close();
-      System.out.println(input.getClass().getSimpleName() + ".close()");
+      if (VERBOSE) System.out.println(input.getClass().getSimpleName() + ".close()");
     }
 
     @Override
     public void reset() throws IOException {
       super.reset();
-      System.out.println(input.getClass().getSimpleName() + ".reset()");
+      if (VERBOSE) System.out.println(input.getClass().getSimpleName() + ".reset()");
     }
   }
   

lucene/analysis/stempel/src/test/org/egothor/stemmer/TestCompile.java

@@ -70,8 +70,9 @@ import java.util.Locale;
 import java.util.StringTokenizer;
 
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 
+@SuppressSysoutChecks(bugUrl = "External tool.")
 public class TestCompile extends LuceneTestCase {
   
   public void testCompile() throws Exception {

lucene/analysis/uima/src/test/org/apache/lucene/analysis/uima/UIMABaseAnalyzerTest.java

@@ -31,6 +31,7 @@ import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -41,6 +42,7 @@ import java.util.Map;
 /**
  * Testcase for {@link UIMABaseAnalyzer}
  */
+@SuppressSysoutChecks(bugUrl = "UIMA logs via ju.logging")
 public class UIMABaseAnalyzerTest extends BaseTokenStreamTestCase {
 
   private UIMABaseAnalyzer analyzer;

lucene/analysis/uima/src/test/org/apache/lucene/analysis/uima/UIMATypeAwareAnalyzerTest.java

@@ -19,6 +19,7 @@ package org.apache.lucene.analysis.uima;
 
 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -26,6 +27,7 @@ import org.junit.Test;
 /**
  * Testcase for {@link UIMATypeAwareAnalyzer}
  */
+@SuppressSysoutChecks(bugUrl = "UIMA logs via ju.logging")
 public class UIMATypeAwareAnalyzerTest extends BaseTokenStreamTestCase {
 
   private UIMATypeAwareAnalyzer analyzer;

lucene/benchmark/src/test/org/apache/lucene/benchmark/BenchmarkTestCase.java

@@ -26,11 +26,12 @@ import java.io.StringReader;
 
 import org.apache.lucene.benchmark.byTask.Benchmark;
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 /** Base class for all Benchmark unit tests. */
+@SuppressSysoutChecks(bugUrl = "Output expected.")
 public abstract class BenchmarkTestCase extends LuceneTestCase {
   private static File WORKDIR;
   

lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java

@@ -62,10 +62,12 @@ import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.TestUtil;
 import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 
 /**
  * Test very simply that perf tasks - simple algorithms - are doing what they should.
  */
+@SuppressSysoutChecks(bugUrl = "Output expected.")
 public class TestPerfTasksLogic extends BenchmarkTestCase {
 
   @Override

lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksParse.java

@@ -37,13 +37,12 @@ import org.apache.lucene.benchmark.byTask.utils.Config;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
-
-import com.carrotsearch.randomizedtesting.RandomizedTest;
 
 import conf.ConfLoader;
 
 /** Test very simply that perf tasks are parses as expected. */
+@SuppressSysoutChecks(bugUrl = "Output expected.")
 public class TestPerfTasksParse extends LuceneTestCase {
 
   static final String NEW_LINE = System.getProperty("line.separator");
@@ -52,8 +51,7 @@ public class TestPerfTasksParse extends LuceneTestCase {
   // properties in effect in all tests here
   static final String propPart = 
     INDENT + "directory=RAMDirectory" + NEW_LINE +
-    INDENT + "print.props=false" + NEW_LINE
+    INDENT + "print.props=false" + NEW_LINE;
-  ;
 
   /** Test the repetiotion parsing for parallel tasks */
   public void testParseParallelTaskSequenceRepetition() throws Exception {

lucene/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java

@@ -26,6 +26,7 @@ import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -43,6 +44,7 @@ import java.nio.charset.StandardCharsets;
  * this test will not work correctly, as it does not dynamically
  * generate its test trec topics/qrels!
  */
+@SuppressSysoutChecks(bugUrl = "Output expected.")
 public class TestQualityRun extends BenchmarkTestCase {
   
   @Override

lucene/common-build.xml

@@ -118,7 +118,7 @@
   <property name="tests.verbose" value="false"/>
   <property name="tests.infostream" value="${tests.verbose}"/>
   <property name="tests.filterstacks" value="true"/>
-  
+
   <condition property="tests.heapsize" value="768M">
     <isset property="run.clover"/>
   </condition>
@@ -1008,6 +1008,7 @@
                 <propertyref prefix="tests.leavetmpdir" />
                 <propertyref prefix="tests.leaveTemporary" />
                 <propertyref prefix="tests.leavetemporary" />
+            	<propertyref prefix="tests.sysouts" />
                 <propertyref prefix="solr.test.leavetmpdir" />
             </syspropertyset>
 

lucene/core/src/java/org/apache/lucene/index/IndexUpgrader.java

@@ -22,6 +22,7 @@ import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.util.CommandLineUtil;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.InfoStream;
+import org.apache.lucene.util.PrintStreamInfoStream;
 import org.apache.lucene.util.Version;
 
 import java.io.File;
@@ -42,7 +43,7 @@ import java.util.Collection;
   * refuses to run by default. Specify {@code -delete-prior-commits}
   * to override this, allowing the tool to delete all but the last commit.
   * From Java code this can be enabled by passing {@code true} to
-  * {@link #IndexUpgrader(Directory,Version,PrintStream,boolean)}.
+  * {@link #IndexUpgrader(Directory,Version,InfoStream,boolean)}.
   * <p><b>Warning:</b> This tool may reorder documents if the index was partially
   * upgraded before execution (e.g., documents were added). If your application relies
   * on &quot;monotonicity&quot; of doc IDs (which means that the order in which the documents
@@ -76,7 +77,7 @@ public final class IndexUpgrader {
   static IndexUpgrader parseArgs(String[] args) throws IOException {
     String path = null;
     boolean deletePriorCommits = false;
-    PrintStream out = null;
+    InfoStream out = null;
     String dirImpl = null;
     int i = 0;
     while (i<args.length) {
@@ -84,7 +85,7 @@ public final class IndexUpgrader {
       if ("-delete-prior-commits".equals(arg)) {
         deletePriorCommits = true;
       } else if ("-verbose".equals(arg)) {
-        out = System.out;
+        out = new PrintStreamInfoStream(System.out);
       } else if ("-dir-impl".equals(arg)) {
         if (i == args.length - 1) {
           System.out.println("ERROR: missing value for -dir-impl option");
@@ -125,7 +126,7 @@ public final class IndexUpgrader {
   /** Creates index upgrader on the given directory, using an {@link IndexWriter} using the given
    * {@code matchVersion}. You have the possibility to upgrade indexes with multiple commit points by removing
    * all older ones. If {@code infoStream} is not {@code null}, all logging output will be sent to this stream. */
-  public IndexUpgrader(Directory dir, Version matchVersion, PrintStream infoStream, boolean deletePriorCommits) {
+  public IndexUpgrader(Directory dir, Version matchVersion, InfoStream infoStream, boolean deletePriorCommits) {
     this(dir, new IndexWriterConfig(matchVersion, null), deletePriorCommits);
     if (null != infoStream) {
       this.iwc.setInfoStream(infoStream);

lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java

@@ -61,6 +61,7 @@ import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.InfoStream;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.apache.lucene.util.StringHelper;
@@ -213,7 +214,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
     switch (choice) {
       case 0: return new IndexUpgrader(dir, TEST_VERSION_CURRENT);
       case 1: return new IndexUpgrader(dir, TEST_VERSION_CURRENT, 
-                                       streamType ? null : System.err, false);
+                                       streamType ? null : InfoStream.NO_OUTPUT, false);
       case 2: return new IndexUpgrader(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null), false);
       default: fail("case statement didn't get updated when random bounds changed");
     }
@@ -876,47 +877,53 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
 
   public void testCommandLineArgs() throws Exception {
 
-    for (String name : oldIndexDirs.keySet()) {
+    PrintStream savedSystemOut = System.out;
-      File dir = createTempDir(name);
+    System.setOut(new PrintStream(new ByteArrayOutputStream(), false, "UTF-8"));
-      File dataFile = new File(TestBackwardsCompatibility.class.getResource("index." + name + ".zip").toURI());
+    try {
-      TestUtil.unzip(dataFile, dir);
+      for (String name : oldIndexDirs.keySet()) {
-
+        File dir = createTempDir(name);
-      String path = dir.getAbsolutePath();
+        File dataFile = new File(TestBackwardsCompatibility.class.getResource("index." + name + ".zip").toURI());
-      
+        TestUtil.unzip(dataFile, dir);
-      List<String> args = new ArrayList<>();
+        
-      if (random().nextBoolean()) {
+        String path = dir.getAbsolutePath();
-        args.add("-verbose");
+        
-      }
+        List<String> args = new ArrayList<>();
-      if (random().nextBoolean()) {
+        if (random().nextBoolean()) {
-        args.add("-delete-prior-commits");
+          args.add("-verbose");
-      }
+        }
-      if (random().nextBoolean()) {
+        if (random().nextBoolean()) {
-        // TODO: need to better randomize this, but ...
+          args.add("-delete-prior-commits");
-        //  - LuceneTestCase.FS_DIRECTORIES is private
+        }
-        //  - newFSDirectory returns BaseDirectoryWrapper
+        if (random().nextBoolean()) {
-        //  - BaseDirectoryWrapper doesn't expose delegate
+          // TODO: need to better randomize this, but ...
-        Class<? extends FSDirectory> dirImpl = random().nextBoolean() ?
+          //  - LuceneTestCase.FS_DIRECTORIES is private
-          SimpleFSDirectory.class : NIOFSDirectory.class;
+          //  - newFSDirectory returns BaseDirectoryWrapper
-
+          //  - BaseDirectoryWrapper doesn't expose delegate
-        args.add("-dir-impl");
+          Class<? extends FSDirectory> dirImpl = random().nextBoolean() ?
-        args.add(dirImpl.getName());
+              SimpleFSDirectory.class : NIOFSDirectory.class;
-      }
+          
-      args.add(path);
+          args.add("-dir-impl");
-
+          args.add(dirImpl.getName());
-      IndexUpgrader upgrader = null;
+        }
-      try {
+        args.add(path);
-        upgrader = IndexUpgrader.parseArgs(args.toArray(new String[0]));
+        
-      } catch (Exception e) {
+        IndexUpgrader upgrader = null;
-        throw new AssertionError("unable to parse args: " + args, e);
+        try {
-      }
+          upgrader = IndexUpgrader.parseArgs(args.toArray(new String[0]));
-      upgrader.upgrade();
+        } catch (Exception e) {
-      
+          throw new AssertionError("unable to parse args: " + args, e);
-      Directory upgradedDir = newFSDirectory(dir);
+        }
-      try {
+        upgrader.upgrade();
-        checkAllSegmentsUpgraded(upgradedDir);
+        
-      } finally {
+        Directory upgradedDir = newFSDirectory(dir);
-        upgradedDir.close();
+        try {
+          checkAllSegmentsUpgraded(upgradedDir);
+        } finally {
+          upgradedDir.close();
+        }
       }
+    } finally {
+      System.setOut(savedSystemOut);
     }
   }
 

lucene/core/src/test/org/apache/lucene/index/TestCrash.java

@@ -93,9 +93,13 @@ public class TestCrash extends LuceneTestCase {
     // This test relies on being able to open a reader before any commit
     // happened, so we must create an initial commit just to allow that, but
     // before any documents were added.
-    System.out.println("TEST: initIndex");
+    if (VERBOSE) {
+      System.out.println("TEST: initIndex");
+    }
     IndexWriter writer = initIndex(random(), true);
-    System.out.println("TEST: done initIndex");
+    if (VERBOSE) {
+      System.out.println("TEST: done initIndex");
+    }
     MockDirectoryWrapper dir = (MockDirectoryWrapper) writer.getDirectory();
 
     // We create leftover files because merging could be
@@ -103,7 +107,9 @@ public class TestCrash extends LuceneTestCase {
     dir.setAssertNoUnrefencedFilesOnClose(false);
 
     dir.setPreventDoubleWrite(false);
-    System.out.println("TEST: now crash");
+    if (VERBOSE) {
+      System.out.println("TEST: now crash");
+    }
     crash(writer);
     writer = initIndex(random(), dir, false);
     writer.shutdown();

lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java

@@ -279,8 +279,10 @@ public class TestDocValuesIndexing extends LuceneTestCase {
       fail("didn't hit expected exception");
     } catch (IllegalArgumentException expected) {
       // expected
-      System.out.println("hit exc:");
+      if (VERBOSE) {
-      expected.printStackTrace(System.out);
+        System.out.println("hit exc:");
+        expected.printStackTrace(System.out);
+      }
     }
     
     iwriter.shutdown();

lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java

@@ -17,8 +17,11 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.PrintStream;
 import java.io.StringReader;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -990,6 +993,8 @@ public class TestIndexWriter extends LuceneTestCase {
     volatile boolean allowInterrupt = false;
     final Random random;
     final Directory adder;
+    final ByteArrayOutputStream bytesLog = new ByteArrayOutputStream();
+    final PrintStream log = new PrintStream(bytesLog, true, IOUtils.UTF_8);
     
     IndexerThreadInterrupt() throws IOException {
       this.random = new Random(random().nextLong());
@@ -1126,27 +1131,27 @@ public class TestIndexWriter extends LuceneTestCase {
           // on!!  This test doesn't repro easily so when
           // Jenkins hits a fail we need to study where the
           // interrupts struck!
-          System.out.println("TEST: got interrupt");
+          log.println("TEST: got interrupt");
-          re.printStackTrace(System.out);
+          re.printStackTrace(log);
           Throwable e = re.getCause();
           assertTrue(e instanceof InterruptedException);
           if (finish) {
             break;
           }
         } catch (Throwable t) {
-          System.out.println("FAILED; unexpected exception");
+          log.println("FAILED; unexpected exception");
-          t.printStackTrace(System.out);
+          t.printStackTrace(log);
           failed = true;
           break;
         }
       }
 
       if (VERBOSE) {
-        System.out.println("TEST: now finish failed=" + failed);
+        log.println("TEST: now finish failed=" + failed);
       }
       if (!failed) {
         if (VERBOSE) {
-          System.out.println("TEST: now rollback");
+          log.println("TEST: now rollback");
         }
         // clear interrupt state:
         Thread.interrupted();
@@ -1162,8 +1167,8 @@ public class TestIndexWriter extends LuceneTestCase {
           TestUtil.checkIndex(dir);
         } catch (Exception e) {
           failed = true;
-          System.out.println("CheckIndex FAILED: unexpected exception");
+          log.println("CheckIndex FAILED: unexpected exception");
-          e.printStackTrace(System.out);
+          e.printStackTrace(log);
         }
         try {
           IndexReader r = DirectoryReader.open(dir);
@@ -1171,8 +1176,8 @@ public class TestIndexWriter extends LuceneTestCase {
           r.close();
         } catch (Exception e) {
           failed = true;
-          System.out.println("DirectoryReader.open FAILED: unexpected exception");
+          log.println("DirectoryReader.open FAILED: unexpected exception");
-          e.printStackTrace(System.out);
+          e.printStackTrace(log);
         }
       }
       try {
@@ -1216,7 +1221,9 @@ public class TestIndexWriter extends LuceneTestCase {
     }
     t.finish = true;
     t.join();
-    assertFalse(t.failed);
+    if (t.failed) {
+      fail(new String(t.bytesLog.toString("UTF-8")));
+    }
   }
   
   /** testThreadInterruptDeadlock but with 2 indexer threads */
@@ -2499,7 +2506,9 @@ public class TestIndexWriter extends LuceneTestCase {
       fail("didn't hit exception");
     } catch (RuntimeException re) {
       // expected
-      System.out.println("GOT: " + re.getMessage());
+      if (VERBOSE) {
+        System.out.println("GOT: " + re.getMessage());
+      }
       assertTrue(re.getMessage().contains("this writer is closed, but some pending changes or running merges were discarded"));
     }
     w.rollback();

lucene/core/src/test/org/apache/lucene/util/junitcompat/TestExceptionInBeforeClassHooks.java

@@ -106,7 +106,7 @@ public class TestExceptionInBeforeClassHooks extends WithNestedTests {
   @Test
   public void testExceptionInBeforeClassFailsTheTest() {
     Result runClasses = JUnitCore.runClasses(Nested1.class);
-    Assert.assertEquals(1, runClasses.getFailureCount());
+    assertFailureCount(1, runClasses);
     Assert.assertEquals(1, runClasses.getRunCount());
     Assert.assertTrue(runClasses.getFailures().get(0).getTrace().contains("foobar"));
   }
@@ -114,7 +114,7 @@ public class TestExceptionInBeforeClassHooks extends WithNestedTests {
   @Test
   public void testExceptionWithinTestFailsTheTest() {
     Result runClasses = JUnitCore.runClasses(Nested2.class);
-    Assert.assertEquals(3, runClasses.getFailureCount());
+    assertFailureCount(3, runClasses);
     Assert.assertEquals(3, runClasses.getRunCount());
     
     ArrayList<String> foobars = new ArrayList<>();
@@ -133,7 +133,7 @@ public class TestExceptionInBeforeClassHooks extends WithNestedTests {
   @Test
   public void testExceptionWithinBefore() {
     Result runClasses = JUnitCore.runClasses(Nested3.class);
-    Assert.assertEquals(1, runClasses.getFailureCount());
+    assertFailureCount(1, runClasses);
     Assert.assertEquals(1, runClasses.getRunCount());
     Assert.assertTrue(runClasses.getFailures().get(0).getTrace().contains("foobar"));
   }  

lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfDirectoryNotClosed.java

@@ -18,11 +18,11 @@ package org.apache.lucene.util.junitcompat;
  */
 
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.JUnitCore;
 import org.junit.runner.Result;
-import org.junit.runner.notification.Failure;
 
 import com.carrotsearch.randomizedtesting.RandomizedTest;
 
@@ -31,6 +31,7 @@ public class TestFailIfDirectoryNotClosed extends WithNestedTests {
     super(true);
   }
   
+  @SuppressSysoutChecks(bugUrl = "Expected.")
   public static class Nested1 extends WithNestedTests.AbstractNestedTest {
     public void testDummy() throws Exception {
       Directory dir = newDirectory();
@@ -43,11 +44,7 @@ public class TestFailIfDirectoryNotClosed extends WithNestedTests {
     Result r = JUnitCore.runClasses(Nested1.class);
     RandomizedTest.assumeTrue("Ignoring nested test, very likely zombie threads present.", 
         r.getIgnoreCount() == 0);
-
+    assertFailureCount(1, r);
-    for (Failure f : r.getFailures()) {
-      System.out.println("Failure: " + f);
-    }
-    Assert.assertEquals(1, r.getFailureCount());
     Assert.assertTrue(r.getFailures().get(0).toString().contains("Resource in scope SUITE failed to close"));
   }
 }

lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSameRandomnessLocalePassedOrNot.java

@@ -3,6 +3,7 @@ package org.apache.lucene.util.junitcompat;
 import java.util.*;
 
 import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 import org.junit.*;
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
@@ -42,6 +43,7 @@ public class TestSameRandomnessLocalePassedOrNot extends WithNestedTests {
     super(true);
   }
   
+  @SuppressSysoutChecks(bugUrl = "Expected.")
   public static class Nested extends WithNestedTests.AbstractNestedTest {
     public static String pickString;
     public static Locale defaultLocale;

lucene/core/src/test/org/apache/lucene/util/junitcompat/TestSeedFromUncaught.java

@@ -18,6 +18,7 @@ package org.apache.lucene.util.junitcompat;
  */
 
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.JUnitCore;
@@ -29,6 +30,7 @@ import org.junit.runner.notification.Failure;
  * console. 
  */
 public class TestSeedFromUncaught extends WithNestedTests {
+  @SuppressSysoutChecks(bugUrl = "Expected.")
   public static class ThrowInUncaught extends AbstractNestedTest {
     @Test
     public void testFoo() throws Exception {
@@ -53,7 +55,7 @@ public class TestSeedFromUncaught extends WithNestedTests {
   @Test
   public void testUncaughtDumpsSeed() {
     Result result = JUnitCore.runClasses(ThrowInUncaught.class);
-    Assert.assertEquals(1, result.getFailureCount());
+    assertFailureCount(1, result);
     Failure f = result.getFailures().get(0);
     String trace = f.getTrace();
     Assert.assertTrue(trace.contains("SeedInfo.seed("));

lucene/core/src/test/org/apache/lucene/util/junitcompat/WithNestedTests.java

@@ -29,6 +29,7 @@ import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures;
 import org.apache.lucene.util.TestRuleIgnoreTestSuites;
 import org.apache.lucene.util.TestRuleMarkFailure;
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Assume;
@@ -37,6 +38,8 @@ import org.junit.ClassRule;
 import org.junit.Rule;
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
+import org.junit.runner.Result;
+import org.junit.runner.notification.Failure;
 
 import com.carrotsearch.randomizedtesting.RandomizedRunner;
 import com.carrotsearch.randomizedtesting.RandomizedTest;
@@ -57,6 +60,7 @@ import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
  * cause havoc (static fields).
  */
 public abstract class WithNestedTests {
+  @SuppressSysoutChecks(bugUrl = "WithNestedTests has its own stream capture.")
   public static abstract class AbstractNestedTest extends LuceneTestCase 
     implements TestRuleIgnoreTestSuites.NestedTestSuite {
     protected static boolean isRunningNested() {
@@ -164,6 +168,20 @@ public abstract class WithNestedTests {
     }
   }
 
+  protected void assertFailureCount(int expected, Result result) {
+    if (result.getFailureCount() != expected) {
+      StringBuilder b = new StringBuilder();
+      for (Failure f : result.getFailures()) {
+        b.append("\n\n");
+        b.append(f.getMessage());
+        b.append("\n");
+        b.append(f.getTrace());
+      }
+      RandomizedTest.assertFalse("Expected failures: " + expected + " but was " + 
+          result.getFailureCount() + ", failures below: " + b.toString(), true);
+    }
+  }
+
   protected String getSysOut() {
     Assert.assertTrue(suppressOutputStreams);
     System.out.flush();

lucene/demo/src/test/org/apache/lucene/demo/TestDemo.java

@@ -23,8 +23,9 @@ import java.io.PrintStream;
 import java.nio.charset.Charset;
 
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 
+@SuppressSysoutChecks(bugUrl = "Output expected.")
 public class TestDemo extends LuceneTestCase {
 
   private void testOneSearch(File indexPath, String query, int expectedHitCount) throws Exception {

lucene/facet/src/test/org/apache/lucene/facet/TestDrillDownQuery.java

@@ -133,7 +133,6 @@ public class TestDrillDownQuery extends FacetTestCase {
     // Making sure the query yields 25 documents with the facet "a"
     DrillDownQuery q = new DrillDownQuery(config);
     q.add("a");
-    System.out.println("q=" + q);
     QueryUtils.check(q);
     TopDocs docs = searcher.search(q, 100);
     assertEquals(25, docs.totalHits);

lucene/misc/src/test/org/apache/lucene/document/TestLazyDocument.java

@@ -17,9 +17,6 @@
 package org.apache.lucene.document;
 
 import java.util.Arrays;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
 import java.util.Set;
 import java.util.HashSet;
 import java.util.Map;
@@ -28,7 +25,6 @@ import java.io.IOException;
 
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.store.*;
-import org.apache.lucene.document.*;
 import org.apache.lucene.analysis.*;
 import org.apache.lucene.index.*;
 import org.apache.lucene.search.*;
@@ -118,7 +114,7 @@ public class TestLazyDocument extends LuceneTestCase {
           assertFalse(f.name() + " is loaded", lf.hasBeenLoaded());
         }
       }
-      System.out.println("numFieldValues == " + numFieldValues);
+      if (VERBOSE) System.out.println("numFieldValues == " + numFieldValues);
       assertEquals("numFieldValues", 1 + (NUM_VALUES * FIELDS.length), 
                    numFieldValues);
         

lucene/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java

@@ -22,7 +22,9 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 
+@SuppressSysoutChecks(bugUrl = "Output expected (external tool).")
 public class TestMultiPassIndexSplitter extends LuceneTestCase {
   IndexReader input;
   int NUM_DOCS = 11;

lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java

@@ -246,6 +246,13 @@ public abstract class LuceneTestCase extends Assert {
 
   /** @see #ignoreAfterMaxFailures*/
   public static final String SYSPROP_FAILFAST = "tests.failfast";
+  
+  /**
+   * If true, enables assertions on writing to system streams.
+   * 
+   * @see TestRuleDisallowSysouts
+   */
+  public static final String SYSPROP_SYSOUTS = "tests.sysouts";
 
   /**
    * Annotation for tests that should only be run during nightly builds.
@@ -346,6 +353,21 @@ public abstract class LuceneTestCase extends Assert {
     public String bugUrl() default "None";
   }
 
+  /**
+   * Marks any suite which is known to print to {@link System#out} or {@link System#err},
+   * even when {@link #VERBOSE} is disabled.
+   * 
+   * @see TestRuleDisallowSysouts
+   */
+  @Documented
+  @Inherited
+  @Retention(RetentionPolicy.RUNTIME)
+  @Target(ElementType.TYPE)
+  public @interface SuppressSysoutChecks {
+    /** Point to JIRA entry. */
+    public String bugUrl();
+  }
+  
   // -----------------------------------------------------------------
   // Truly immutable fields and constants, initialized once and valid 
   // for all suites ever since.
@@ -360,11 +382,13 @@ public abstract class LuceneTestCase extends Assert {
 
   /**
    * True if and only if tests are run in verbose mode. If this flag is false
-   * tests are not expected to print any messages.
+   * tests are not expected to print any messages. Enforced with {@link TestRuleDisallowSysouts}.
    */
   public static final boolean VERBOSE = systemPropertyAsBoolean("tests.verbose", false);
 
-  /** TODO: javadoc? */
+  /**
+   * Enables or disables dumping of {@link InfoStream} messages. 
+   */
   public static final boolean INFOSTREAM = systemPropertyAsBoolean("tests.infostream", VERBOSE);
 
   /**
@@ -478,7 +502,7 @@ public abstract class LuceneTestCase extends Assert {
   /**
    * Suite failure marker (any error in the test or suite scope).
    */
-  public static TestRuleMarkFailure suiteFailureMarker;
+  private static TestRuleMarkFailure suiteFailureMarker;
 
   /**
    * Ignore tests after hitting a designated number of initial failures. This
@@ -511,6 +535,15 @@ public abstract class LuceneTestCase extends Assert {
             new TestRuleIgnoreAfterMaxFailures(maxFailures));
     ignoreAfterMaxFailures = TestRuleDelegate.of(ignoreAfterMaxFailuresDelegate);
   }
+  
+  /**
+   * Try to capture streams early so that other classes don't have a chance to steal references
+   * to them (as is the case with ju.logging handlers).
+   */
+  static {
+    TestRuleDisallowSysouts.checkCaptureStreams();
+    Logger.getGlobal().getHandlers();
+  }
 
   /**
    * Temporarily substitute the global {@link TestRuleIgnoreAfterMaxFailures}. See
@@ -545,6 +578,7 @@ public abstract class LuceneTestCase extends Assert {
     .around(ignoreAfterMaxFailures)
     .around(suiteFailureMarker = new TestRuleMarkFailure())
     .around(new TestRuleAssertionsRequired())
+    .around(new TestRuleDisallowSysouts(suiteFailureMarker))
     .around(new TemporaryFilesCleanupRule())
     .around(new StaticFieldsInvariantRule(STATIC_LEAK_THRESHOLD, true) {
       @Override
@@ -2359,6 +2393,12 @@ public abstract class LuceneTestCase extends Assert {
     }
   }
 
+  /**
+   * Checks and cleans up temporary files.
+   * 
+   * @see LuceneTestCase#createTempDir()
+   * @see LuceneTestCase#createTempFile()
+   */
   private static class TemporaryFilesCleanupRule extends TestRuleAdapter {
     @Override
     protected void before() throws Throwable {

lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java

@@ -154,6 +154,9 @@ public final class RunListenerPrintReproduceInfo extends RunListener {
     addVmOpt(b, "testcase", RandomizedContext.current().getTargetClass().getSimpleName());
     addVmOpt(b, "tests.method", testName);
     addVmOpt(b, "tests.seed", RandomizedContext.current().getRunnerSeedAsString());
+    
+    // Misc switches.
+    addVmOpt(b, SYSPROP_SYSOUTS, System.getProperty(SYSPROP_SYSOUTS));
 
     // Test groups and multipliers.
     if (RANDOM_MULTIPLIER > 1) addVmOpt(b, "tests.multiplier", RANDOM_MULTIPLIER);

lucene/test-framework/src/java/org/apache/lucene/util/TestRuleDisallowSysouts.java

@@ -0,0 +1,193 @@
+package org.apache.lucene.util;
+
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
+
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
+
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Fails the suite if it prints anything to {@link System#out} or {@link System#err},
+ * unless the condition is not enforced (see {@link #isEnforced()}).
+ */
+public class TestRuleDisallowSysouts extends TestRuleAdapter {
+  /** 
+   * Stack trace of any thread that wrote something to sysout or syserr. 
+   */
+  private final static AtomicReference<StackTraceElement[]> firstWriteStack = new AtomicReference<StackTraceElement[]>();
+
+  private final static DelegateStream capturedSystemOut;
+  private final static DelegateStream capturedSystemErr;
+  
+  static {
+    System.out.flush();
+    System.err.flush();
+
+    final String csn = Charset.defaultCharset().name();
+    capturedSystemOut = new DelegateStream(System.out, csn, firstWriteStack);
+    capturedSystemErr = new DelegateStream(System.err, csn, firstWriteStack);
+
+    System.setOut(capturedSystemOut.printStream);
+    System.setErr(capturedSystemErr.printStream);
+  }
+
+  /**
+   * Test failures from any tests or rules before.
+   */
+  private final TestRuleMarkFailure failureMarker;
+
+  /**
+   * Sets {@link #firstWriteStack} to the current stack trace upon the first actual write
+   * to an underlying stream.
+   */
+  static class DelegateStream extends FilterOutputStream {
+    private final AtomicReference<StackTraceElement[]> firstWriteStack;
+    final PrintStream printStream;
+
+    public DelegateStream(OutputStream delegate, String charset, AtomicReference<StackTraceElement[]> firstWriteStack) {
+      super(delegate);
+      try {
+        this.firstWriteStack = firstWriteStack;
+        this.printStream = new PrintStream(this, true, charset);
+      } catch (UnsupportedEncodingException e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    // Do override all three write() methods to make sure nothing slips through.
+
+    @Override
+    public void write(byte[] b) throws IOException {
+      if (b.length > 0) {
+        bytesWritten();
+      }
+      super.write(b);
+    }
+    
+    @Override
+    public void write(byte[] b, int off, int len) throws IOException {
+      if (len > 0) {
+        bytesWritten();
+      }
+      super.write(b, off, len);
+    }
+
+    @Override
+    public void write(int b) throws IOException {
+      bytesWritten();
+      super.write(b);
+    }
+    
+    private void bytesWritten() {
+      // This check isn't really needed, but getting the stack is expensive and may involve
+      // jit deopts, so we'll do it anyway.
+      if (firstWriteStack.get() == null) {
+        firstWriteStack.compareAndSet(null, Thread.currentThread().getStackTrace());
+      }
+    }
+  }
+  
+  public TestRuleDisallowSysouts(TestRuleMarkFailure failureMarker) {
+    this.failureMarker = failureMarker;
+  }
+
+  
+  /** */
+  @Override
+  protected void before() throws Throwable {
+    if (isEnforced()) {
+      checkCaptureStreams();
+    }
+    resetCaptureState();
+  }
+
+  /**
+   * Ensures {@link System#out} and {@link System#err} point to delegate streams.
+   */
+  public static void checkCaptureStreams() {
+    // Make sure we still hold the right references to wrapper streams.
+    if (System.out != capturedSystemOut.printStream) {
+      throw new AssertionError("Something has changed System.out to: " + System.out.getClass().getName());
+    }
+    if (System.err != capturedSystemErr.printStream) {
+      throw new AssertionError("Something has changed System.err to: " + System.err.getClass().getName());
+    }
+  }
+
+  protected boolean isEnforced() {
+    if (LuceneTestCase.VERBOSE || 
+        LuceneTestCase.INFOSTREAM ||
+        RandomizedTest.getContext().getTargetClass().isAnnotationPresent(SuppressSysoutChecks.class)) {
+      return false;
+    }
+    
+    return !RandomizedTest.systemPropertyAsBoolean(LuceneTestCase.SYSPROP_SYSOUTS, true);
+  }
+
+  /**
+   * We're only interested in failing the suite if it was successful. Otherwise
+   * just propagate the original problem and don't bother.
+   */
+  @Override
+  protected void afterIfSuccessful() throws Throwable {
+    if (isEnforced()) {
+      checkCaptureStreams();
+  
+      // Flush any buffers.
+      capturedSystemOut.printStream.flush();
+      capturedSystemErr.printStream.flush();
+  
+      // And check for offenders, but only if everything was successful so far.
+      StackTraceElement[] offenderStack = firstWriteStack.get();
+      if (offenderStack != null && failureMarker.wasSuccessful()) {
+        AssertionError e = new AssertionError("The test or suite printed information to stdout or stderr," +
+            " even though verbose mode is turned off and it's not annotated with @" + 
+            SuppressSysoutChecks.class.getSimpleName() + ". This exception contains the stack" +
+                " trace of the first offending call.");
+        e.setStackTrace(offenderStack);
+        throw e;
+      }
+    }
+  }
+
+  /**
+   * Restore original streams.
+   */
+  @Override
+  protected void afterAlways(List<Throwable> errors) throws Throwable {
+    resetCaptureState();
+  }
+
+  private void resetCaptureState() {
+    capturedSystemOut.printStream.flush();
+    capturedSystemErr.printStream.flush();
+    firstWriteStack.set(null);
+  }  
+}
+

solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java

@@ -58,6 +58,7 @@ import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.QuickPatchThreadsFilter;
 import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 import org.apache.solr.client.solrj.impl.HttpClientConfigurer;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
 import org.apache.solr.client.solrj.util.ClientUtils;
@@ -118,6 +119,7 @@ import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;
     SolrIgnoredThreadsFilter.class,
     QuickPatchThreadsFilter.class
 })
+@SuppressSysoutChecks(bugUrl = "Solr dumps logs to console.")
 public abstract class SolrTestCaseJ4 extends LuceneTestCase {
   private static String coreName = ConfigSolrXmlOld.DEFAULT_DEFAULT_CORE_NAME;
   public static int DEFAULT_CONNECTION_TIMEOUT = 60000;  // default socket connection timeout in ms