From 24a2bc3f62848a3afc212aba5138dea9b854f352 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 13 Jan 2021 16:20:39 +0000 Subject: [PATCH] Extend the selective muting of memory tests on Debian 8 (#67455) The selective muting implemented for autoscaling in #67159 is extended to the ML tests that also fail when machine memory is reported as 0. Most of the logic to determine when memory will not be accurately reported is now in a utility method in the base class. Relates #66885 Backport of #67422 --- .../elasticsearch/test/ESIntegTestCase.java | 20 +++++++++++++++++++ .../integration/BasicDistributedJobsIT.java | 4 ++++ .../integration/MlDistributedFailureIT.java | 3 +++ .../xpack/ml/integration/TooManyJobsIT.java | 4 ++++ 4 files changed, 31 insertions(+) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index ae1b9049b3a..ced0bc8003e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -57,6 +57,7 @@ import org.elasticsearch.action.search.ClearScrollResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.bootstrap.JavaVersion; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.Client; import org.elasticsearch.client.ClusterAdminClient; @@ -129,6 +130,7 @@ import org.elasticsearch.indices.IndicesQueryCache; import org.elasticsearch.indices.IndicesRequestCache; import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.ingest.IngestMetadata; +import org.elasticsearch.monitor.os.OsInfo; import org.elasticsearch.node.NodeMocksPlugin; import org.elasticsearch.plugins.NetworkPlugin; import org.elasticsearch.plugins.Plugin; @@ -2392,4 +2394,22 @@ public abstract class ESIntegTestCase extends ESTestCase { public static boolean inFipsJvm() { return Boolean.parseBoolean(System.getProperty(FIPS_SYSPROP)); } + + /** + * On Debian 8 the "memory" subsystem is not mounted by default + * when cgroups are enabled, and this confuses many versions of + * Java prior to Java 15. Tests that rely on machine memory + * being accurately determined will not work on such setups, + * and can use this method for selective muting. + * See https://github.com/elastic/elasticsearch/issues/67089 + * and https://github.com/elastic/elasticsearch/issues/66885 + */ + protected boolean willSufferDebian8MemoryProblem() { + final NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().execute().actionGet(); + final boolean anyDebian8Nodes = response.getNodes() + .stream() + .anyMatch(ni -> ni.getInfo(OsInfo.class).getPrettyName().equals("Debian GNU/Linux 8 (jessie)")); + boolean java15Plus = JavaVersion.current().compareTo(JavaVersion.parse("15")) >= 0; + return anyDebian8Nodes && java15Plus == false; + } } diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java index 5619c3b6890..c735a91395c 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java @@ -423,6 +423,10 @@ public class BasicDistributedJobsIT extends BaseMlIntegTestCase { } public void testCloseUnassignedLazyJobAndDatafeed() throws Exception { + + // see: https://github.com/elastic/elasticsearch/issues/66885#issuecomment-758790179 + assumeFalse("cannot run on debian 8 prior to java 15", willSufferDebian8MemoryProblem()); + internalCluster().ensureAtLeastNumDataNodes(3); ensureStableCluster(3); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java index dc0af77dc8a..5b1c1b52051 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java @@ -373,6 +373,9 @@ public class MlDistributedFailureIT extends BaseMlIntegTestCase { public void testJobRelocationIsMemoryAware() throws Exception { + // see: https://github.com/elastic/elasticsearch/issues/66885#issuecomment-758790179 + assumeFalse("cannot run on debian 8 prior to java 15", willSufferDebian8MemoryProblem()); + internalCluster().ensureAtLeastNumDataNodes(1); ensureStableCluster(); diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java index 569aee4c261..044cb712f49 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java @@ -126,10 +126,14 @@ public class TooManyJobsIT extends BaseMlIntegTestCase { } public void testSingleNode() throws Exception { + // see: https://github.com/elastic/elasticsearch/issues/66885#issuecomment-758790179 + assumeFalse("cannot run on debian 8 prior to java 15", willSufferDebian8MemoryProblem()); verifyMaxNumberOfJobsLimit(1, randomIntBetween(1, 20), randomBoolean()); } public void testMultipleNodes() throws Exception { + // see: https://github.com/elastic/elasticsearch/issues/66885#issuecomment-758790179 + assumeFalse("cannot run on debian 8 prior to java 15", willSufferDebian8MemoryProblem()); verifyMaxNumberOfJobsLimit(3, randomIntBetween(1, 20), randomBoolean()); }