[TEST] Randomize number of available processors

We configure the threadpools according to the number of processors which is
different on every machine. Yet, we had some test failures related to this
and #6174 that only happened reproducibly on a node with 1 available processor.
This commit does:
  * sometimes randomize the number of available processors
  * if we don't randomize we should set the actual number of available processors
    in the settings on the test node
  * always print out the num of processors when a test fails to make sure we can
    reproduce the thread pool settings with the reproduce info line

Closes #6176

pom.xml

@@ -458,6 +458,7 @@
                                 <java.io.tmpdir>.</java.io.tmpdir> <!-- we use '.' since this is different per JVM-->
                                 <!-- RandomizedTesting library system properties -->
                                 <tests.jvm.argline>${tests.jvm.argline}</tests.jvm.argline>
+                                <tests.processors>${tests.processors}</tests.processors>
                                 <tests.appendseed>${tests.appendseed}</tests.appendseed>
                                 <tests.iters>${tests.iters}</tests.iters>
                                 <tests.maxfailures>${tests.maxfailures}</tests.maxfailures>

src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java

@@ -29,6 +29,12 @@ import java.util.concurrent.atomic.AtomicInteger;
  */
 public class EsExecutors {
 
+    /**
+     * Settings key to manually set the number of available processors.
+     * This is used to adjust thread pools sizes etc. per node.
+     */
+    public static final String PROCESSORS = "processors";
+
     /**
      * Returns the number of processors available but at most <tt>32</tt>.
      */
@@ -37,7 +43,7 @@ public class EsExecutors {
          * ie. >= 48 create too many threads and run into OOM see #3478
          * We just use an 32 core upper-bound here to not stress the system
          * too much with too many created threads */
-        return settings.getAsInt("processors", Math.min(32, Runtime.getRuntime().availableProcessors()));
+        return settings.getAsInt(PROCESSORS, Math.min(32, Runtime.getRuntime().availableProcessors()));
     }
 
     public static PrioritizedEsThreadPoolExecutor newSinglePrioritizing(ThreadFactory threadFactory) {

src/test/java/org/apache/lucene/util/AbstractRandomizedTest.java

@@ -32,6 +32,8 @@ import com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule;
 import com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.test.ElasticsearchIntegrationTest;
 import org.elasticsearch.test.ElasticsearchTestCase;
 import org.elasticsearch.test.junit.listeners.ReproduceInfoPrinter;
@@ -90,6 +92,8 @@ public abstract class AbstractRandomizedTest extends RandomizedTest {
 
     public static final String SYSPROP_INTEGRATION = "tests.integration";
 
+    public static final String SYSPROP_PROCESSORS = "tests.processors";
+
     // -----------------------------------------------------------------
     // Truly immutable fields and constants, initialized once and valid 
     // for all suites ever since.
@@ -128,12 +132,20 @@ public abstract class AbstractRandomizedTest extends RandomizedTest {
      */
     public static final File TEMP_DIR;
 
+    public static final int TESTS_PROCESSORS;
+
     static {
         String s = System.getProperty("tempDir", System.getProperty("java.io.tmpdir"));
         if (s == null)
             throw new RuntimeException("To run tests, you need to define system property 'tempDir' or 'java.io.tmpdir'.");
         TEMP_DIR = new File(s);
         TEMP_DIR.mkdirs();
+
+        String processors = System.getProperty(SYSPROP_PROCESSORS, ""); // mvn sets "" as default
+        if (processors == null || processors.isEmpty()) {
+            processors = Integer.toString(EsExecutors.boundedNumberOfProcessors(ImmutableSettings.EMPTY));
+        }
+        TESTS_PROCESSORS = Integer.parseInt(processors);
     }
 
     /**

src/test/java/org/elasticsearch/test/TestCluster.java

@@ -27,6 +27,7 @@ import com.google.common.collect.*;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.SettableFuture;
+import org.apache.lucene.util.AbstractRandomizedTest;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchIllegalStateException;
@@ -302,6 +303,11 @@ public final class TestCluster extends ImmutableTestCluster {
         }
         builder.put("plugins.isolation", random.nextBoolean());
         builder.put(InternalGlobalOrdinalsBuilder.ORDINAL_MAPPING_THRESHOLD_INDEX_SETTING_KEY, 1 + random.nextInt(InternalGlobalOrdinalsBuilder.ORDINAL_MAPPING_THRESHOLD_DEFAULT));
+        if (random.nextInt(10) == 0) {
+            builder.put(EsExecutors.PROCESSORS, 1 + random.nextInt(AbstractRandomizedTest.TESTS_PROCESSORS));
+        } else {
+            builder.put(EsExecutors.PROCESSORS, AbstractRandomizedTest.TESTS_PROCESSORS);
+        }
         return builder.build();
     }
 

src/test/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java

@@ -21,6 +21,7 @@ package org.elasticsearch.test.junit.listeners;
 import com.carrotsearch.randomizedtesting.RandomizedContext;
 import com.carrotsearch.randomizedtesting.ReproduceErrorMessageBuilder;
 import com.carrotsearch.randomizedtesting.TraceFormatting;
+import org.apache.lucene.util.AbstractRandomizedTest;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
@@ -139,6 +140,7 @@ public class ReproduceInfoPrinter extends RunListener {
             if (System.getProperty("tests.jvm.argline") != null && !System.getProperty("tests.jvm.argline").isEmpty()) {
                 appendOpt("tests.jvm.argline", "\"" + System.getProperty("tests.jvm.argline") + "\"");
             }
+            appendOpt(AbstractRandomizedTest.SYSPROP_PROCESSORS, Integer.toString(AbstractRandomizedTest.TESTS_PROCESSORS));
             return this;
         }