From 8c6978c3baef96a333ebd7e98e02098c99df7313 Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Fri, 4 Jan 2019 14:07:02 +0530 Subject: [PATCH] YARN-6149. Allow port range to be specified while starting NM Timeline collector manager. Contributed by Abhishek Modi. --- .../hadoop/yarn/conf/YarnConfiguration.java | 4 ++++ .../collector/NodeTimelineCollectorManager.java | 15 +++++++++++++-- .../collector/TestNMTimelineCollectorManager.java | 6 ++++-- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 511edefd3d1..08f7f1a18a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3868,6 +3868,10 @@ public class YarnConfiguration extends Configuration { public static final String TIMELINE_SERVICE_COLLECTOR_BIND_HOST = TIMELINE_SERVICE_COLLECTOR_PREFIX + "bind-host"; + @Private + public static final String TIMELINE_SERVICE_COLLECTOR_BIND_PORT_RANGES = + TIMELINE_SERVICE_COLLECTOR_PREFIX + "bind-port-ranges"; + @Private public static final String TIMELINE_SERVICE_COLLECTOR_WEBAPP_ADDRESS = TIMELINE_SERVICE_COLLECTOR_PREFIX + "webapp.address"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java index 696f4a3ed30..0371d4997aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java @@ -58,6 +58,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + /** * Class on the NodeManager side that manages adding and removing collectors and * their lifecycle. Also instantiates the per-node collector webapp. @@ -280,14 +281,21 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager { String bindAddress = null; String host = conf.getTrimmed(YarnConfiguration.TIMELINE_SERVICE_COLLECTOR_BIND_HOST); + Configuration.IntegerRanges portRanges = conf.getRange( + YarnConfiguration.TIMELINE_SERVICE_COLLECTOR_BIND_PORT_RANGES, ""); + int startPort = 0; + if (portRanges != null && !portRanges.isEmpty()) { + startPort = portRanges.getRangeStart(); + } if (host == null || host.isEmpty()) { // if collector bind-host is not set, fall back to // timeline-service.bind-host to maintain compatibility bindAddress = conf.get(YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST) + ":0"; + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST) + + ":" + startPort; } else { - bindAddress = host + ":0"; + bindAddress = host + ":" + startPort; } try { @@ -297,6 +305,9 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager { .addEndpoint(URI.create( (YarnConfiguration.useHttps(conf) ? "https://" : "http://") + bindAddress)); + if (portRanges != null && !portRanges.isEmpty()) { + builder.setPortRanges(portRanges); + } if (YarnConfiguration.useHttps(conf)) { builder = WebAppUtils.loadSslConfiguration(builder, conf); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java index af9acce265e..0f2af21ae42 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java @@ -60,6 +60,8 @@ public class TestNMTimelineCollectorManager { Configuration conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS, FileSystemTimelineWriterImpl.class, TimelineWriter.class); + conf.set(YarnConfiguration.TIMELINE_SERVICE_COLLECTOR_BIND_PORT_RANGES, + "30000-30100"); collectorManager.init(conf); collectorManager.start(); } @@ -83,7 +85,8 @@ public class TestNMTimelineCollectorManager { String[] parts = address.split(":"); assertEquals(2, parts.length); assertNotNull(parts[0]); - assertTrue(Integer.valueOf(parts[1]) > 0); + assertTrue(Integer.valueOf(parts[1]) >= 30000 && + Integer.valueOf(parts[1]) <= 30100); } @Test(timeout=60000) @@ -153,7 +156,6 @@ public class TestNMTimelineCollectorManager { private NodeTimelineCollectorManager createCollectorManager() { final NodeTimelineCollectorManager cm = spy(new NodeTimelineCollectorManager()); - doReturn(new Configuration()).when(cm).getConfig(); CollectorNodemanagerProtocol nmCollectorService = mock(CollectorNodemanagerProtocol.class); GetTimelineCollectorContextResponse response =