diff --git a/pom.xml b/pom.xml index ea8629eabe3..83fbb93ef22 100644 --- a/pom.xml +++ b/pom.xml @@ -458,6 +458,7 @@ . ${tests.jvm.argline} + ${tests.processors} ${tests.appendseed} ${tests.iters} ${tests.maxfailures} diff --git a/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java b/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java index c045e1d2e3f..d4fd161cf99 100644 --- a/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java +++ b/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java @@ -29,6 +29,12 @@ import java.util.concurrent.atomic.AtomicInteger; */ public class EsExecutors { + /** + * Settings key to manually set the number of available processors. + * This is used to adjust thread pools sizes etc. per node. + */ + public static final String PROCESSORS = "processors"; + /** * Returns the number of processors available but at most 32. */ @@ -37,7 +43,7 @@ public class EsExecutors { * ie. >= 48 create too many threads and run into OOM see #3478 * We just use an 32 core upper-bound here to not stress the system * too much with too many created threads */ - return settings.getAsInt("processors", Math.min(32, Runtime.getRuntime().availableProcessors())); + return settings.getAsInt(PROCESSORS, Math.min(32, Runtime.getRuntime().availableProcessors())); } public static PrioritizedEsThreadPoolExecutor newSinglePrioritizing(ThreadFactory threadFactory) { diff --git a/src/test/java/org/apache/lucene/util/AbstractRandomizedTest.java b/src/test/java/org/apache/lucene/util/AbstractRandomizedTest.java index 522bdc3d084..ef561054074 100644 --- a/src/test/java/org/apache/lucene/util/AbstractRandomizedTest.java +++ b/src/test/java/org/apache/lucene/util/AbstractRandomizedTest.java @@ -32,6 +32,8 @@ import com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule; import com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.ElasticsearchTestCase; import org.elasticsearch.test.junit.listeners.ReproduceInfoPrinter; @@ -90,6 +92,8 @@ public abstract class AbstractRandomizedTest extends RandomizedTest { public static final String SYSPROP_INTEGRATION = "tests.integration"; + public static final String SYSPROP_PROCESSORS = "tests.processors"; + // ----------------------------------------------------------------- // Truly immutable fields and constants, initialized once and valid // for all suites ever since. @@ -128,12 +132,20 @@ public abstract class AbstractRandomizedTest extends RandomizedTest { */ public static final File TEMP_DIR; + public static final int TESTS_PROCESSORS; + static { String s = System.getProperty("tempDir", System.getProperty("java.io.tmpdir")); if (s == null) throw new RuntimeException("To run tests, you need to define system property 'tempDir' or 'java.io.tmpdir'."); TEMP_DIR = new File(s); TEMP_DIR.mkdirs(); + + String processors = System.getProperty(SYSPROP_PROCESSORS, ""); // mvn sets "" as default + if (processors == null || processors.isEmpty()) { + processors = Integer.toString(EsExecutors.boundedNumberOfProcessors(ImmutableSettings.EMPTY)); + } + TESTS_PROCESSORS = Integer.parseInt(processors); } /** diff --git a/src/test/java/org/elasticsearch/test/TestCluster.java b/src/test/java/org/elasticsearch/test/TestCluster.java index cb07db561a6..8a8e34ca8b3 100644 --- a/src/test/java/org/elasticsearch/test/TestCluster.java +++ b/src/test/java/org/elasticsearch/test/TestCluster.java @@ -27,6 +27,7 @@ import com.google.common.collect.*; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; +import org.apache.lucene.util.AbstractRandomizedTest; import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchIllegalStateException; @@ -302,6 +303,11 @@ public final class TestCluster extends ImmutableTestCluster { } builder.put("plugins.isolation", random.nextBoolean()); builder.put(InternalGlobalOrdinalsBuilder.ORDINAL_MAPPING_THRESHOLD_INDEX_SETTING_KEY, 1 + random.nextInt(InternalGlobalOrdinalsBuilder.ORDINAL_MAPPING_THRESHOLD_DEFAULT)); + if (random.nextInt(10) == 0) { + builder.put(EsExecutors.PROCESSORS, 1 + random.nextInt(AbstractRandomizedTest.TESTS_PROCESSORS)); + } else { + builder.put(EsExecutors.PROCESSORS, AbstractRandomizedTest.TESTS_PROCESSORS); + } return builder.build(); } diff --git a/src/test/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java b/src/test/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java index e987ca9eb30..59a60a98858 100644 --- a/src/test/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java +++ b/src/test/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java @@ -21,6 +21,7 @@ package org.elasticsearch.test.junit.listeners; import com.carrotsearch.randomizedtesting.RandomizedContext; import com.carrotsearch.randomizedtesting.ReproduceErrorMessageBuilder; import com.carrotsearch.randomizedtesting.TraceFormatting; +import org.apache.lucene.util.AbstractRandomizedTest; import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; @@ -139,6 +140,7 @@ public class ReproduceInfoPrinter extends RunListener { if (System.getProperty("tests.jvm.argline") != null && !System.getProperty("tests.jvm.argline").isEmpty()) { appendOpt("tests.jvm.argline", "\"" + System.getProperty("tests.jvm.argline") + "\""); } + appendOpt(AbstractRandomizedTest.SYSPROP_PROCESSORS, Integer.toString(AbstractRandomizedTest.TESTS_PROCESSORS)); return this; }