Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-7662. [ATSv2] Define new set of configurations for reader and collectors to bind (Rohith Sharma K S via Varun Saxena)

This commit is contained in:
Varun Saxena 2017-12-19 22:29:24 +05:30
parent fe5b057c81
commit c0aeb666a4
10 changed files with 145 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -2389,6 +2389,9 @@ public class YarnConfiguration extends Configuration {
/**
* Settings for timeline service v2.0.
*/
public static final String TIMELINE_SERVICE_READER_PREFIX =
TIMELINE_SERVICE_PREFIX + "reader.";
public static final String TIMELINE_SERVICE_WRITER_CLASS =
TIMELINE_SERVICE_PREFIX + "writer.class";
@ -2397,7 +2400,7 @@ public class YarnConfiguration extends Configuration {
+ ".storage.HBaseTimelineWriterImpl";
public static final String TIMELINE_SERVICE_READER_CLASS =
TIMELINE_SERVICE_PREFIX + "reader.class";
TIMELINE_SERVICE_READER_PREFIX + "class";
public static final String DEFAULT_TIMELINE_SERVICE_READER_CLASS =
"org.apache.hadoop.yarn.server.timelineservice.storage" +
@ -3422,6 +3425,46 @@ public class YarnConfiguration extends Configuration {
public static final String TIMELINE_XFS_OPTIONS =
TIMELINE_XFS_PREFIX + "xframe-options";
/**
* Settings for timeline reader.
*/
public static final String TIMELINE_SERVICE_READER_BIND_HOST =
TIMELINE_SERVICE_READER_PREFIX + "bind-host";
public static final String TIMELINE_SERVICE_READER_WEBAPP_ADDRESS =
TIMELINE_SERVICE_READER_PREFIX + "webapp.address";
public static final String DEFAULT_TIMELINE_SERVICE_READER_WEBAPP_ADDRESS =
DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS;
public static final String TIMELINE_SERVICE_READER_WEBAPP_HTTPS_ADDRESS =
TIMELINE_SERVICE_READER_PREFIX + "webapp.https.address";
public static final String
DEFAULT_TIMELINE_SERVICE_READER_WEBAPP_HTTPS_ADDRESS =
DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS;
/**
* Marked collector properties as Private since it run as auxillary service.
*/
public static final String TIMELINE_SERVICE_COLLECTOR_PREFIX =
TIMELINE_SERVICE_PREFIX + "collector.";
@Private
public static final String TIMELINE_SERVICE_COLLECTOR_BIND_HOST =
TIMELINE_SERVICE_COLLECTOR_PREFIX + "bind-host";
@Private
public static final String TIMELINE_SERVICE_COLLECTOR_WEBAPP_ADDRESS =
TIMELINE_SERVICE_COLLECTOR_PREFIX + "webapp.address";
public static final String DEFAULT_TIMELINE_SERVICE_COLLECTOR_WEBAPP_ADDRESS =
DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS;
@Private
public static final String TIMELINE_SERVICE_COLLECTOR_WEBAPP_HTTPS_ADDRESS =
TIMELINE_SERVICE_COLLECTOR_PREFIX + "webapp.https.address";
public static final String
DEFAULT_TIMELINE_SERVICE_COLLECTOR_WEBAPP_HTTPS_ADDRESS =
DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS;
public YarnConfiguration() {
super();
}

39
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
View File

@ -314,16 +314,41 @@ public class WebAppUtils {
}
public static String getAHSWebAppURLWithoutScheme(Configuration conf) {
return getTimelineReaderWebAppURL(conf);
}
public static String getTimelineReaderWebAppURL(Configuration conf) {
if (YarnConfiguration.useHttps(conf)) {
return conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS);
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS);
} else {
return conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS);
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS);
}
}
public static String getTimelineReaderWebAppURLWithoutScheme(
Configuration conf) {
if (YarnConfiguration.useHttps(conf)) {
return conf
.get(YarnConfiguration.TIMELINE_SERVICE_READER_WEBAPP_HTTPS_ADDRESS,
YarnConfiguration.
DEFAULT_TIMELINE_SERVICE_READER_WEBAPP_HTTPS_ADDRESS);
} else {
return conf.get(YarnConfiguration.TIMELINE_SERVICE_READER_WEBAPP_ADDRESS,
YarnConfiguration.
DEFAULT_TIMELINE_SERVICE_READER_WEBAPP_ADDRESS);
}
}
public static String getTimelineCollectorWebAppURLWithoutScheme(
Configuration conf) {
if (YarnConfiguration.useHttps(conf)) {
return conf.get(
YarnConfiguration.TIMELINE_SERVICE_COLLECTOR_WEBAPP_HTTPS_ADDRESS,
YarnConfiguration.
DEFAULT_TIMELINE_SERVICE_COLLECTOR_WEBAPP_HTTPS_ADDRESS);
} else {
return conf
.get(YarnConfiguration.TIMELINE_SERVICE_COLLECTOR_WEBAPP_ADDRESS,
YarnConfiguration.
DEFAULT_TIMELINE_SERVICE_COLLECTOR_WEBAPP_ADDRESS);
}
}
@ -342,7 +367,7 @@ public class WebAppUtils {
return schemePrefix + url;
}
}
public static String getRunningLogURL(
String nodeHttpAddress, String containerId, String user) {
if (nodeHttpAddress == null || nodeHttpAddress.isEmpty() ||

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -3599,4 +3599,28 @@
<value>0,1</value>
</property>
<property>
<description>The http address of the timeline reader web application.</description>
<name>yarn.timeline-service.reader.webapp.address</name>
<value>${yarn.timeline-service.webapp.address}</value>
</property>
<property>
<description>The https address of the timeline reader web application.</description>
<name>yarn.timeline-service.reader.webapp.https.address</name>
<value>${yarn.timeline-service.webapp.https.address}</value>
</property>
<property>
<description>
The actual address timeline reader will bind to. If this optional address is
set, the reader server will bind to this address and the port specified in
yarn.timeline-service.reader.webapp.address.
This is most useful for making the service listen to all interfaces by setting to
0.0.0.0.
</description>
<name>yarn.timeline-service.reader.bind-host</name>
<value></value>
</property>
</configuration>

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java
View File

@ -162,7 +162,8 @@ public class TestTimelineAuthFilterForV2 {
conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
FileSystemTimelineWriterImpl.class, TimelineWriter.class);
conf.set(YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, "localhost");
conf.set(YarnConfiguration.TIMELINE_SERVICE_READER_BIND_HOST,
"localhost");
conf.set(FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT,
TEST_ROOT_DIR.getAbsolutePath());
conf.set("hadoop.proxyuser.HTTP.hosts", "*");

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/AbstractTimelineReaderHBaseTestBase.java
View File

@ -79,7 +79,7 @@ public abstract class AbstractTimelineReaderHBaseTestBase {
Configuration config = util.getConfiguration();
config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
config.set(YarnConfiguration.TIMELINE_SERVICE_READER_WEBAPP_ADDRESS,
"localhost:0");
config.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
config.set(YarnConfiguration.TIMELINE_SERVICE_READER_CLASS,

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
View File

@ -145,10 +145,9 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
private void doSecureLogin() throws IOException {
Configuration conf = getConfig();
InetSocketAddress addr = NetUtils.createSocketAddr(conf.getTrimmed(
YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST), 0,
YarnConfiguration.TIMELINE_SERVICE_BIND_HOST);
String webAppURLWithoutScheme =
WebAppUtils.getTimelineCollectorWebAppURLWithoutScheme(conf);
InetSocketAddress addr = NetUtils.createSocketAddr(webAppURLWithoutScheme);
SecurityUtil.login(conf, YarnConfiguration.TIMELINE_SERVICE_KEYTAB,
YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL, addr.getHostName());
}
@ -277,8 +276,20 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
initializers, defaultInitializers, tokenMgrService);
TimelineServerUtils.setTimelineFilters(
conf, initializers, defaultInitializers);
String bindAddress = conf.get(YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST) + ":0";
String bindAddress = null;
String host =
conf.getTrimmed(YarnConfiguration.TIMELINE_SERVICE_COLLECTOR_BIND_HOST);
if (host == null || host.isEmpty()) {
// if collector bind-host is not set, fall back to
// timeline-service.bind-host to maintain compatibility
bindAddress =
conf.get(YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST) + ":0";
} else {
bindAddress = host + ":0";
}
try {
HttpServer2.Builder builder = new HttpServer2.Builder()
.setName("timeline")

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
View File

@ -28,6 +28,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.service.CompositeService;
@ -63,6 +64,8 @@ public class TimelineReaderServer extends CompositeService {
private HttpServer2 readerWebServer;
private TimelineReaderManager timelineReaderManager;
private String webAppURLWithoutScheme;
public TimelineReaderServer() {
super(TimelineReaderServer.class.getName());
@ -73,10 +76,10 @@ public class TimelineReaderServer extends CompositeService {
if (!YarnConfiguration.timelineServiceV2Enabled(conf)) {
throw new YarnException("timeline service v.2 is not enabled");
}
InetSocketAddress bindAddr = conf.getSocketAddr(
YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT);
webAppURLWithoutScheme =
WebAppUtils.getTimelineReaderWebAppURLWithoutScheme(conf);
InetSocketAddress bindAddr =
NetUtils.createSocketAddr(webAppURLWithoutScheme);
// Login from keytab if security is enabled.
try {
SecurityUtil.login(conf, YarnConfiguration.TIMELINE_SERVICE_KEYTAB,
@ -170,9 +173,17 @@ public class TimelineReaderServer extends CompositeService {
private void startTimelineReaderWebApp() {
Configuration conf = getConfig();
addFilters(conf);
String bindAddress = WebAppUtils.getWebAppBindURL(conf,
YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
WebAppUtils.getTimelineReaderWebAppURL(conf));
String hostProperty = YarnConfiguration.TIMELINE_SERVICE_READER_BIND_HOST;
String host = conf.getTrimmed(hostProperty);
if (host == null || host.isEmpty()) {
// if reader bind-host is not set, fall back to timeline-service.bind-host
// to maintain compatibility
hostProperty = YarnConfiguration.TIMELINE_SERVICE_BIND_HOST;
}
String bindAddress = WebAppUtils
.getWebAppBindURL(conf, hostProperty, webAppURLWithoutScheme);
LOG.info("Instantiating TimelineReaderWebApp at " + bindAddress);
try {
HttpServer2.Builder builder = new HttpServer2.Builder()

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderServer.java
View File

@ -37,7 +37,7 @@ public class TestTimelineReaderServer {
Configuration config = new YarnConfiguration();
config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
config.set(YarnConfiguration.TIMELINE_SERVICE_READER_WEBAPP_ADDRESS,
"localhost:0");
config.setClass(YarnConfiguration.TIMELINE_SERVICE_READER_CLASS,
FileSystemTimelineReaderImpl.class, TimelineReader.class);
@ -61,7 +61,7 @@ public class TestTimelineReaderServer {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
conf.set(YarnConfiguration.TIMELINE_SERVICE_READER_WEBAPP_ADDRESS,
"localhost:0");
conf.set(YarnConfiguration.TIMELINE_SERVICE_READER_CLASS,
Object.class.getName());
@ -75,7 +75,7 @@ public class TestTimelineReaderServer {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
conf.set(YarnConfiguration.TIMELINE_SERVICE_READER_WEBAPP_ADDRESS,
"localhost:0");
conf.set(YarnConfiguration.TIMELINE_SERVICE_READER_CLASS,
nonexistentTimelineReaderClass);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
View File

@ -84,7 +84,7 @@ public class TestTimelineReaderWebServices {
Configuration config = new YarnConfiguration();
config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
config.set(YarnConfiguration.TIMELINE_SERVICE_READER_WEBAPP_ADDRESS,
"localhost:0");
config.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
config.setClass(YarnConfiguration.TIMELINE_SERVICE_READER_CLASS,

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
View File

@ -133,9 +133,9 @@ New configuration parameters that are introduced with v.2 are marked bold.
| Configuration Property | Description |
|:---- |:---- |
| `yarn.timeline-service.hostname` | The hostname of the Timeline service web application. Defaults to `0.0.0.0` |
| `yarn.timeline-service.address` | Address for the Timeline server to start the RPC server. Defaults to `${yarn.timeline-service.hostname}:10200`. |
| `yarn.timeline-service.webapp.address` | The http address of the Timeline service web application. Defaults to `${yarn.timeline-service.hostname}:8188`. |
| `yarn.timeline-service.webapp.https.address` | The https address of the Timeline service web application. Defaults to `${yarn.timeline-service.hostname}:8190`. |
| `yarn.timeline-service.reader.webapp.address` | The http address of the Timeline Reader web application. Defaults to `${yarn.timeline-service.hostname}:8188`. |
| `yarn.timeline-service.reader.webapp.https.address` | The https address of the Timeline Reader web application. Defaults to `${yarn.timeline-service.hostname}:8190`. |
| `yarn.timeline-service.reader.bind-host` | The actual address the timeline reader will bind to. If this optional address is set, reader server will bind to this address and the port specified in yarn.timeline-service.reader.webapp.address. This is most useful for making the service listen on all interfaces by setting to 0.0.0.0. |
| **`yarn.timeline-service.hbase.configuration.file`** | Optional URL to an hbase-site.xml configuration file to be used to connect to the timeline-service hbase cluster. If empty or not specified, then the HBase configuration will be loaded from the classpath. When specified the values in the specified configuration file will override those from the ones that are present on the classpath. Defaults to `null`. |
| **`yarn.timeline-service.writer.flush-interval-seconds`** | The setting that controls how often the timeline collector flushes the timeline writer. Defaults to `60`. |
| **`yarn.timeline-service.app-collector.linger-period.ms`** | Time period till which the application collector will be alive in NM, after the application master container finishes. Defaults to `1000` (1 second). |