diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java index 01c70b296eb..4f6846b5c44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java @@ -46,6 +46,8 @@ import org.apache.hadoop.yarn.util.Records; public abstract class NodeAttribute { public static final String DEFAULT_PREFIX = ""; + public static final String PREFIX_DISTRIBUTED = "nm.yarn.io"; + public static final String PREFIX_CENTRALIZED = "rm.yarn.io"; public static NodeAttribute newInstance(String attributeName, NodeAttributeType attributeType, String attributeValue) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java index d9187124d09..fdfd0ce7033 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java @@ -17,7 +17,11 @@ */ package org.apache.hadoop.yarn.nodelabels; +import com.google.common.base.Strings; +import org.apache.hadoop.yarn.api.records.NodeAttribute; + import java.io.IOException; +import java.util.Set; import java.util.regex.Pattern; /** @@ -94,4 +98,31 @@ public final class NodeLabelUtil { + ", now it is= " + prefix); } } + + /** + * Validate if a given set of attributes are valid. Attributes could be + * invalid if any of following conditions is met: + * + * + * @param attributeSet + * @throws IOException + */ + public static void validateNodeAttributes(Set attributeSet) + throws IOException { + if (attributeSet != null && !attributeSet.isEmpty()) { + for (NodeAttribute nodeAttribute : attributeSet) { + String prefix = nodeAttribute.getAttributePrefix(); + if (Strings.isNullOrEmpty(prefix)) { + throw new IOException("Attribute prefix must be set"); + } + // Verify attribute prefix format. + checkAndThrowAttributePrefix(prefix); + // Verify attribute name format. + checkAndThrowLabelName(nodeAttribute.getAttributeName()); + } + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java index 06771bae28c..46214348997 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.nodemanager.nodelabels; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.NodeAttribute; import org.apache.hadoop.yarn.api.records.NodeAttributeType; +import org.apache.hadoop.yarn.nodelabels.NodeLabelUtil; import java.io.IOException; import java.util.HashSet; @@ -116,13 +117,33 @@ public class ScriptBasedNodeAttributesProvider extends NodeAttributesProvider{ + NODE_ATTRIBUTE_DELIMITER + "ATTRIBUTE_VALUE; but get " + nodeAttribute); } + // Automatically setup prefix for collected attributes NodeAttribute na = NodeAttribute - .newInstance(attributeStrs[0], + .newInstance(NodeAttribute.PREFIX_DISTRIBUTED, + attributeStrs[0], NodeAttributeType.valueOf(attributeStrs[1]), attributeStrs[2]); - attributeSet.add(na); + + // Since a NodeAttribute is identical with another one as long as + // their prefix and name are same, to avoid attributes getting + // overwritten by ambiguous attribute, make sure it fails in such + // case. + if (!attributeSet.add(na)) { + throw new IOException("Ambiguous node attribute is found: " + + na.toString() + ", a same attribute already exists"); + } } } + + // Before updating the attributes to the provider, + // verify if they are valid + try { + NodeLabelUtil.validateNodeAttributes(attributeSet); + } catch (IOException e) { + throw new IOException("Node attributes collected by the script " + + "contains some invalidate entries. Detail message: " + + e.getMessage()); + } return attributeSet; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java index 58d2d20916b..f764626b4eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java @@ -220,4 +220,31 @@ public class TestScriptBasedNodeAttributesProvider { } }, 500, 3000); } + + @Test + public void testNodeAttributesValidation() throws Exception{ + // Script output contains ambiguous node attributes + String scriptContent = "echo NODE_ATTRIBUTE:host,STRING,host1234\n " + + "echo NODE_ATTRIBUTE:host,STRING,host2345\n " + + "echo NODE_ATTRIBUTE:ip,STRING,10.0.0.1"; + + writeNodeAttributeScriptFile(scriptContent, true); + + nodeAttributesProvider.init(getConfForNodeAttributeScript()); + nodeAttributesProvider.start(); + + // There should be no attributes found, and we should + // see Malformed output warnings in the log + try { + GenericTestUtils + .waitFor(() -> nodeAttributesProvider + .getDescriptors().size() == 3, + 500, 3000); + Assert.fail("This test should timeout because the provide is unable" + + " to parse any attributes from the script output."); + } catch (TimeoutException e) { + Assert.assertEquals(0, nodeAttributesProvider + .getDescriptors().size()); + } + } }