From 179090c84035d3351c2c647c34d38f9a52b5c50d Mon Sep 17 00:00:00 2001 From: David Kyle Date: Thu, 22 Mar 2018 17:45:57 +0000 Subject: [PATCH] [ML] Unclutter failed job assignment explanations (elastic/x-pack-elasticsearch#4179) Unclutter failed job assignment explanations Original commit: elastic/x-pack-elasticsearch@1c3deebaace1dbe328e602fe24fd857b009c03e6 --- .../ml/action/TransportOpenJobAction.java | 59 ++++++++++++++----- .../action/TransportOpenJobActionTests.java | 29 +++++++++ 2 files changed, 74 insertions(+), 14 deletions(-) diff --git a/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java index 7cac57d841c..658c3a5b607 100644 --- a/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java +++ b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CheckedSupplier; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -154,7 +155,8 @@ public class TransportOpenJobAction extends TransportMasterNodeAction nodeAttributes = node.getAttributes(); String enabled = nodeAttributes.get(MachineLearning.ML_ENABLED_NODE_ATTR); if (Boolean.valueOf(enabled) == false) { - String reason = "Not opening job [" + jobId + "] on node [" + node + "], because this node isn't a ml node."; + String reason = "Not opening job [" + jobId + "] on node [" + nodeNameOrId(node) + + "], because this node isn't a ml node."; logger.trace(reason); reasons.add(reason); continue; @@ -164,15 +166,15 @@ public class TransportOpenJobAction extends TransportMasterNodeAction compatibleJobTypes = Job.getCompatibleJobTypes(node.getVersion()); if (compatibleJobTypes.contains(job.getJobType()) == false) { - String reason = "Not opening job [" + jobId + "] on node [" + node + "], because this node does not support jobs of type [" - + job.getJobType() + "]"; + String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndVersion(node) + + "], because this node does not support jobs of type [" + job.getJobType() + "]"; logger.trace(reason); reasons.add(reason); continue; } if (nodeSupportsJobVersion(node.getVersion()) == false) { - String reason = "Not opening job [" + jobId + "] on node [" + node + String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndVersion(node) + "], because this node does not support jobs of version [" + job.getJobVersion() + "]"; logger.trace(reason); reasons.add(reason); @@ -180,8 +182,9 @@ public class TransportOpenJobAction extends TransportMasterNodeAction= maxConcurrentJobAllocations) { - String reason = "Not opening job [" + jobId + "] on node [" + node + "], because node exceeds [" + numberOfAllocatingJobs + + String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndMlAttributes(node) + + "], because node exceeds [" + numberOfAllocatingJobs + "] the maximum number of jobs [" + maxConcurrentJobAllocations + "] in opening state"; logger.trace(reason); reasons.add(reason); @@ -224,7 +228,7 @@ public class TransportOpenJobAction extends TransportMasterNodeAction availableMemory) { - String reason = "Not opening job [" + jobId + "] on node [" + node + + String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndMlAttributes(node) + "], because this node has insufficient available memory. Available memory for ML [" + maxMlMemory + "], memory required by existing jobs [" + assignedJobMemory + "], estimated memory required for this job [" + estimatedMemoryFootprint + "]"; @@ -285,7 +289,7 @@ public class TransportOpenJobAction extends TransportMasterNodeAction entry : node.getAttributes().entrySet()) { + if (entry.getKey().startsWith("ml.") || entry.getKey().equals("node.ml")) { + builder.append('{').append(entry).append('}'); + } + } + return builder.toString(); + } + static String[] indicesOfInterest(ClusterState clusterState, String job) { String jobResultIndex = AnomalyDetectorsIndex.getPhysicalIndexFromState(clusterState, job); return new String[]{AnomalyDetectorsIndex.jobStateIndexName(), jobResultIndex, MlMetaIndex.INDEX_NAME}; diff --git a/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java b/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java index ba9fb8775bc..a0e0570a3f1 100644 --- a/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java +++ b/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java @@ -54,6 +54,8 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; import java.util.function.Function; import static org.elasticsearch.xpack.core.ml.job.config.JobTests.buildJobBuilder; @@ -496,6 +498,33 @@ public class TransportOpenJobActionTests extends ESTestCase { assertArrayEquals(indices, TransportOpenJobAction.mappingRequiresUpdate(cs, indices, Version.CURRENT, logger)); } + public void testNodeNameAndVersion() { + TransportAddress ta = new TransportAddress(InetAddress.getLoopbackAddress(), 9300); + Map attributes = new HashMap<>(); + attributes.put("unrelated", "attribute"); + DiscoveryNode node = new DiscoveryNode("_node_name1", "_node_id1", ta, attributes, Collections.emptySet(), Version.CURRENT); + assertEquals("{_node_name1}{version=" + node.getVersion() + "}", TransportOpenJobAction.nodeNameAndVersion(node)); + } + + public void testNodeNameAndMlAttributes() { + TransportAddress ta = new TransportAddress(InetAddress.getLoopbackAddress(), 9300); + SortedMap attributes = new TreeMap<>(); + attributes.put("unrelated", "attribute"); + DiscoveryNode node = new DiscoveryNode("_node_name1", "_node_id1", ta, attributes, Collections.emptySet(), Version.CURRENT); + assertEquals("{_node_name1}", TransportOpenJobAction.nodeNameAndMlAttributes(node)); + + attributes.put("ml.machine_memory", "5"); + node = new DiscoveryNode("_node_name1", "_node_id1", ta, attributes, Collections.emptySet(), Version.CURRENT); + assertEquals("{_node_name1}{ml.machine_memory=5}", TransportOpenJobAction.nodeNameAndMlAttributes(node)); + + node = new DiscoveryNode(null, "_node_id1", ta, attributes, Collections.emptySet(), Version.CURRENT); + assertEquals("{_node_id1}{ml.machine_memory=5}", TransportOpenJobAction.nodeNameAndMlAttributes(node)); + + attributes.put("node.ml", "true"); + node = new DiscoveryNode("_node_name1", "_node_id1", ta, attributes, Collections.emptySet(), Version.CURRENT); + assertEquals("{_node_name1}{ml.machine_memory=5}{node.ml=true}", TransportOpenJobAction.nodeNameAndMlAttributes(node)); + } + public static void addJobTask(String jobId, String nodeId, JobState jobState, PersistentTasksCustomMetaData.Builder builder) { builder.addTask(MlMetadata.jobTaskId(jobId), OpenJobAction.TASK_NAME, new OpenJobAction.JobParams(jobId), new Assignment(nodeId, "test assignment"));