YARN-1942. Deprecate toString/fromString methods from ConverterUtils and move them to records classes like ContainerId/ApplicationId, etc. (wangda)
This commit is contained in:
parent
9c66fffe4a
commit
863bfa4d6a
|
@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
|
||||||
import org.apache.hadoop.yarn.api.ApplicationConstants;
|
import org.apache.hadoop.yarn.api.ApplicationConstants;
|
||||||
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
|
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -291,8 +292,7 @@ class YarnChild {
|
||||||
Credentials credentials, Token<JobTokenIdentifier> jt) throws IOException {
|
Credentials credentials, Token<JobTokenIdentifier> jt) throws IOException {
|
||||||
job.setCredentials(credentials);
|
job.setCredentials(credentials);
|
||||||
|
|
||||||
ApplicationAttemptId appAttemptId =
|
ApplicationAttemptId appAttemptId = ContainerId.fromString(
|
||||||
ConverterUtils.toContainerId(
|
|
||||||
System.getenv(Environment.CONTAINER_ID.name()))
|
System.getenv(Environment.CONTAINER_ID.name()))
|
||||||
.getApplicationAttemptId();
|
.getApplicationAttemptId();
|
||||||
LOG.debug("APPLICATION_ATTEMPT_ID: " + appAttemptId);
|
LOG.debug("APPLICATION_ATTEMPT_ID: " + appAttemptId);
|
||||||
|
|
|
@ -1544,7 +1544,7 @@ public class MRAppMaster extends CompositeService {
|
||||||
validateInputParam(appSubmitTimeStr,
|
validateInputParam(appSubmitTimeStr,
|
||||||
ApplicationConstants.APP_SUBMIT_TIME_ENV);
|
ApplicationConstants.APP_SUBMIT_TIME_ENV);
|
||||||
|
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
ApplicationAttemptId applicationAttemptId =
|
ApplicationAttemptId applicationAttemptId =
|
||||||
containerId.getApplicationAttemptId();
|
containerId.getApplicationAttemptId();
|
||||||
if (applicationAttemptId != null) {
|
if (applicationAttemptId != null) {
|
||||||
|
|
|
@ -721,8 +721,7 @@ public abstract class TaskAttemptImpl implements
|
||||||
LocalResourceType type, LocalResourceVisibility visibility)
|
LocalResourceType type, LocalResourceVisibility visibility)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
FileStatus fstat = fc.getFileStatus(file);
|
FileStatus fstat = fc.getFileStatus(file);
|
||||||
URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
|
URL resourceURL = URL.fromPath(fc.resolvePath(fstat.getPath()));
|
||||||
.getPath()));
|
|
||||||
long resourceSize = fstat.getLen();
|
long resourceSize = fstat.getLen();
|
||||||
long resourceModificationTime = fstat.getModificationTime();
|
long resourceModificationTime = fstat.getModificationTime();
|
||||||
|
|
||||||
|
@ -1263,8 +1262,8 @@ public abstract class TaskAttemptImpl implements
|
||||||
public TaskAttemptStateInternal recover(TaskAttemptInfo taInfo,
|
public TaskAttemptStateInternal recover(TaskAttemptInfo taInfo,
|
||||||
OutputCommitter committer, boolean recoverOutput) {
|
OutputCommitter committer, boolean recoverOutput) {
|
||||||
ContainerId containerId = taInfo.getContainerId();
|
ContainerId containerId = taInfo.getContainerId();
|
||||||
NodeId containerNodeId = ConverterUtils.toNodeId(taInfo.getHostname() + ":"
|
NodeId containerNodeId = NodeId.fromString(
|
||||||
+ taInfo.getPort());
|
taInfo.getHostname() + ":" + taInfo.getPort());
|
||||||
String nodeHttpAddress = StringInterner.weakIntern(taInfo.getHostname() + ":"
|
String nodeHttpAddress = StringInterner.weakIntern(taInfo.getHostname() + ":"
|
||||||
+ taInfo.getHttpPort());
|
+ taInfo.getHttpPort());
|
||||||
// Resource/Priority/Tokens are only needed while launching the container on
|
// Resource/Priority/Tokens are only needed while launching the container on
|
||||||
|
|
|
@ -69,8 +69,10 @@ public class TaskAttemptInfo {
|
||||||
this.nodeHttpAddress = ta.getNodeHttpAddress();
|
this.nodeHttpAddress = ta.getNodeHttpAddress();
|
||||||
this.startTime = report.getStartTime();
|
this.startTime = report.getStartTime();
|
||||||
this.finishTime = report.getFinishTime();
|
this.finishTime = report.getFinishTime();
|
||||||
this.assignedContainerId = ConverterUtils.toString(report.getContainerId());
|
|
||||||
this.assignedContainer = report.getContainerId();
|
this.assignedContainer = report.getContainerId();
|
||||||
|
if (assignedContainer != null) {
|
||||||
|
this.assignedContainerId = assignedContainer.toString();
|
||||||
|
}
|
||||||
this.progress = report.getProgress() * 100;
|
this.progress = report.getProgress() * 100;
|
||||||
this.status = report.getStateString();
|
this.status = report.getStateString();
|
||||||
this.state = report.getTaskAttemptState();
|
this.state = report.getTaskAttemptState();
|
||||||
|
|
|
@ -253,7 +253,7 @@ public class MRApp extends MRAppMaster {
|
||||||
// the job can reaches the final state when MRAppMaster shuts down.
|
// the job can reaches the final state when MRAppMaster shuts down.
|
||||||
this.successfullyUnregistered.set(unregistered);
|
this.successfullyUnregistered.set(unregistered);
|
||||||
this.assignedQueue = assignedQueue;
|
this.assignedQueue = assignedQueue;
|
||||||
this.resource = Resource.newInstance(1234, 2);
|
this.resource = Resource.newInstance(1234L, 2L);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -136,9 +136,9 @@ public class TestMRAppMaster {
|
||||||
String containerIdStr = "container_1317529182569_0004_000001_1";
|
String containerIdStr = "container_1317529182569_0004_000001_1";
|
||||||
|
|
||||||
String userName = "TestAppMasterUser";
|
String userName = "TestAppMasterUser";
|
||||||
ApplicationAttemptId applicationAttemptId = ConverterUtils
|
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId(applicationAttemptIdStr);
|
applicationAttemptIdStr);
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
MRAppMasterTest appMaster =
|
MRAppMasterTest appMaster =
|
||||||
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
||||||
System.currentTimeMillis());
|
System.currentTimeMillis());
|
||||||
|
@ -159,15 +159,15 @@ public class TestMRAppMaster {
|
||||||
String userName = "TestAppMasterUser";
|
String userName = "TestAppMasterUser";
|
||||||
JobConf conf = new JobConf();
|
JobConf conf = new JobConf();
|
||||||
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
||||||
ApplicationAttemptId applicationAttemptId = ConverterUtils
|
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId(applicationAttemptIdStr);
|
applicationAttemptIdStr);
|
||||||
JobId jobId = TypeConverter.toYarn(
|
JobId jobId = TypeConverter.toYarn(
|
||||||
TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
|
TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
|
||||||
Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
|
Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
|
||||||
FileSystem fs = FileSystem.get(conf);
|
FileSystem fs = FileSystem.get(conf);
|
||||||
//Create the file, but no end file so we should unregister with an error.
|
//Create the file, but no end file so we should unregister with an error.
|
||||||
fs.create(start).close();
|
fs.create(start).close();
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
MRAppMaster appMaster =
|
MRAppMaster appMaster =
|
||||||
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
||||||
System.currentTimeMillis(), false, false);
|
System.currentTimeMillis(), false, false);
|
||||||
|
@ -198,8 +198,8 @@ public class TestMRAppMaster {
|
||||||
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
||||||
conf.setInt(MRJobConfig.NUM_REDUCES, 0);
|
conf.setInt(MRJobConfig.NUM_REDUCES, 0);
|
||||||
conf.set(JHAdminConfig.MR_HS_JHIST_FORMAT, "json");
|
conf.set(JHAdminConfig.MR_HS_JHIST_FORMAT, "json");
|
||||||
ApplicationAttemptId applicationAttemptId = ConverterUtils
|
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId(applicationAttemptIdStr);
|
applicationAttemptIdStr);
|
||||||
JobId jobId = TypeConverter.toYarn(
|
JobId jobId = TypeConverter.toYarn(
|
||||||
TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
|
TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
|
||||||
|
|
||||||
|
@ -217,7 +217,7 @@ public class TestMRAppMaster {
|
||||||
FileSystem fs = FileSystem.get(conf);
|
FileSystem fs = FileSystem.get(conf);
|
||||||
JobSplitWriter.createSplitFiles(new Path(dir.getAbsolutePath()), conf,
|
JobSplitWriter.createSplitFiles(new Path(dir.getAbsolutePath()), conf,
|
||||||
fs, new org.apache.hadoop.mapred.InputSplit[0]);
|
fs, new org.apache.hadoop.mapred.InputSplit[0]);
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
MRAppMasterTestLaunchTime appMaster =
|
MRAppMasterTestLaunchTime appMaster =
|
||||||
new MRAppMasterTestLaunchTime(applicationAttemptId, containerId,
|
new MRAppMasterTestLaunchTime(applicationAttemptId, containerId,
|
||||||
"host", -1, -1, System.currentTimeMillis());
|
"host", -1, -1, System.currentTimeMillis());
|
||||||
|
@ -235,8 +235,8 @@ public class TestMRAppMaster {
|
||||||
String userName = "TestAppMasterUser";
|
String userName = "TestAppMasterUser";
|
||||||
JobConf conf = new JobConf();
|
JobConf conf = new JobConf();
|
||||||
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
||||||
ApplicationAttemptId applicationAttemptId = ConverterUtils
|
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId(applicationAttemptIdStr);
|
applicationAttemptIdStr);
|
||||||
JobId jobId = TypeConverter.toYarn(
|
JobId jobId = TypeConverter.toYarn(
|
||||||
TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
|
TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
|
||||||
Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
|
Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
|
||||||
|
@ -244,7 +244,7 @@ public class TestMRAppMaster {
|
||||||
FileSystem fs = FileSystem.get(conf);
|
FileSystem fs = FileSystem.get(conf);
|
||||||
fs.create(start).close();
|
fs.create(start).close();
|
||||||
fs.create(end).close();
|
fs.create(end).close();
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
MRAppMaster appMaster =
|
MRAppMaster appMaster =
|
||||||
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
||||||
System.currentTimeMillis(), false, false);
|
System.currentTimeMillis(), false, false);
|
||||||
|
@ -273,8 +273,8 @@ public class TestMRAppMaster {
|
||||||
String userName = "TestAppMasterUser";
|
String userName = "TestAppMasterUser";
|
||||||
JobConf conf = new JobConf();
|
JobConf conf = new JobConf();
|
||||||
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
||||||
ApplicationAttemptId applicationAttemptId = ConverterUtils
|
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId(applicationAttemptIdStr);
|
applicationAttemptIdStr);
|
||||||
JobId jobId = TypeConverter.toYarn(
|
JobId jobId = TypeConverter.toYarn(
|
||||||
TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
|
TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
|
||||||
Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
|
Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
|
||||||
|
@ -282,7 +282,7 @@ public class TestMRAppMaster {
|
||||||
FileSystem fs = FileSystem.get(conf);
|
FileSystem fs = FileSystem.get(conf);
|
||||||
fs.create(start).close();
|
fs.create(start).close();
|
||||||
fs.create(end).close();
|
fs.create(end).close();
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
MRAppMaster appMaster =
|
MRAppMaster appMaster =
|
||||||
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
||||||
System.currentTimeMillis(), false, false);
|
System.currentTimeMillis(), false, false);
|
||||||
|
@ -311,8 +311,8 @@ public class TestMRAppMaster {
|
||||||
String userName = "TestAppMasterUser";
|
String userName = "TestAppMasterUser";
|
||||||
JobConf conf = new JobConf();
|
JobConf conf = new JobConf();
|
||||||
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
||||||
ApplicationAttemptId applicationAttemptId = ConverterUtils
|
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId(applicationAttemptIdStr);
|
applicationAttemptIdStr);
|
||||||
|
|
||||||
//Delete the staging directory
|
//Delete the staging directory
|
||||||
File dir = new File(stagingDir);
|
File dir = new File(stagingDir);
|
||||||
|
@ -320,7 +320,7 @@ public class TestMRAppMaster {
|
||||||
FileUtils.deleteDirectory(dir);
|
FileUtils.deleteDirectory(dir);
|
||||||
}
|
}
|
||||||
|
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
MRAppMaster appMaster =
|
MRAppMaster appMaster =
|
||||||
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
||||||
System.currentTimeMillis(), false, false);
|
System.currentTimeMillis(), false, false);
|
||||||
|
@ -351,9 +351,9 @@ public class TestMRAppMaster {
|
||||||
String containerIdStr = "container_1317529182569_0004_000002_1";
|
String containerIdStr = "container_1317529182569_0004_000002_1";
|
||||||
|
|
||||||
String userName = "TestAppMasterUser";
|
String userName = "TestAppMasterUser";
|
||||||
ApplicationAttemptId applicationAttemptId = ConverterUtils
|
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId(applicationAttemptIdStr);
|
applicationAttemptIdStr);
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
JobConf conf = new JobConf();
|
JobConf conf = new JobConf();
|
||||||
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
||||||
|
|
||||||
|
@ -505,9 +505,9 @@ public class TestMRAppMaster {
|
||||||
String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
|
String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
|
||||||
String containerIdStr = "container_1317529182569_0004_000002_1";
|
String containerIdStr = "container_1317529182569_0004_000002_1";
|
||||||
String userName = "TestAppMasterUser";
|
String userName = "TestAppMasterUser";
|
||||||
ApplicationAttemptId applicationAttemptId = ConverterUtils
|
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId(applicationAttemptIdStr);
|
applicationAttemptIdStr);
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
JobConf conf = new JobConf();
|
JobConf conf = new JobConf();
|
||||||
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
||||||
|
|
||||||
|
|
|
@ -129,8 +129,8 @@ public class TestCommitterEventHandler {
|
||||||
|
|
||||||
SystemClock clock = SystemClock.getInstance();
|
SystemClock clock = SystemClock.getInstance();
|
||||||
AppContext appContext = mock(AppContext.class);
|
AppContext appContext = mock(AppContext.class);
|
||||||
ApplicationAttemptId attemptid =
|
ApplicationAttemptId attemptid = ApplicationAttemptId.fromString(
|
||||||
ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
|
"appattempt_1234567890000_0001_0");
|
||||||
when(appContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
|
when(appContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
|
||||||
when(appContext.getApplicationAttemptId()).thenReturn(attemptid);
|
when(appContext.getApplicationAttemptId()).thenReturn(attemptid);
|
||||||
when(appContext.getEventHandler()).thenReturn(
|
when(appContext.getEventHandler()).thenReturn(
|
||||||
|
@ -240,8 +240,8 @@ public class TestCommitterEventHandler {
|
||||||
YarnConfiguration conf = new YarnConfiguration();
|
YarnConfiguration conf = new YarnConfiguration();
|
||||||
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
||||||
JobContext mockJobContext = mock(JobContext.class);
|
JobContext mockJobContext = mock(JobContext.class);
|
||||||
ApplicationAttemptId attemptid =
|
ApplicationAttemptId attemptid = ApplicationAttemptId.fromString(
|
||||||
ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
|
"appattempt_1234567890000_0001_0");
|
||||||
JobId jobId = TypeConverter.toYarn(
|
JobId jobId = TypeConverter.toYarn(
|
||||||
TypeConverter.fromYarn(attemptid.getApplicationId()));
|
TypeConverter.fromYarn(attemptid.getApplicationId()));
|
||||||
|
|
||||||
|
@ -289,7 +289,7 @@ public class TestCommitterEventHandler {
|
||||||
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
||||||
JobContext mockJobContext = mock(JobContext.class);
|
JobContext mockJobContext = mock(JobContext.class);
|
||||||
ApplicationAttemptId attemptid =
|
ApplicationAttemptId attemptid =
|
||||||
ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
|
ApplicationAttemptId.fromString("appattempt_1234567890000_0001_0");
|
||||||
JobId jobId = TypeConverter.toYarn(
|
JobId jobId = TypeConverter.toYarn(
|
||||||
TypeConverter.fromYarn(attemptid.getApplicationId()));
|
TypeConverter.fromYarn(attemptid.getApplicationId()));
|
||||||
|
|
||||||
|
|
|
@ -942,8 +942,8 @@ public class TestJobImpl {
|
||||||
callback.run();
|
callback.run();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
ApplicationAttemptId id =
|
ApplicationAttemptId id = ApplicationAttemptId.fromString(
|
||||||
ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
|
"appattempt_1234567890000_0001_0");
|
||||||
when(appContext.getApplicationID()).thenReturn(id.getApplicationId());
|
when(appContext.getApplicationID()).thenReturn(id.getApplicationId());
|
||||||
when(appContext.getApplicationAttemptId()).thenReturn(id);
|
when(appContext.getApplicationAttemptId()).thenReturn(id);
|
||||||
CommitterEventHandler handler =
|
CommitterEventHandler handler =
|
||||||
|
|
|
@ -515,7 +515,7 @@ public class TestAMWebServicesAttempts extends JerseyTest {
|
||||||
WebServicesTestUtils.checkStringMatch("diagnostics", expectDiag,
|
WebServicesTestUtils.checkStringMatch("diagnostics", expectDiag,
|
||||||
diagnostics);
|
diagnostics);
|
||||||
WebServicesTestUtils.checkStringMatch("assignedContainerId",
|
WebServicesTestUtils.checkStringMatch("assignedContainerId",
|
||||||
ConverterUtils.toString(ta.getAssignedContainerID()),
|
ta.getAssignedContainerID().toString(),
|
||||||
assignedContainerId);
|
assignedContainerId);
|
||||||
|
|
||||||
assertEquals("startTime wrong", ta.getLaunchTime(), startTime);
|
assertEquals("startTime wrong", ta.getLaunchTime(), startTime);
|
||||||
|
|
|
@ -157,7 +157,7 @@ class LocalDistributedCacheManager {
|
||||||
}
|
}
|
||||||
Path resourcePath;
|
Path resourcePath;
|
||||||
try {
|
try {
|
||||||
resourcePath = ConverterUtils.getPathFromYarnURL(resource.getResource());
|
resourcePath = resource.getResource().toPath();
|
||||||
} catch (URISyntaxException e) {
|
} catch (URISyntaxException e) {
|
||||||
throw new IOException(e);
|
throw new IOException(e);
|
||||||
}
|
}
|
||||||
|
|
|
@ -68,6 +68,7 @@ import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResource;
|
import org.apache.hadoop.yarn.api.records.LocalResource;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResourceType;
|
import org.apache.hadoop.yarn.api.records.LocalResourceType;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
||||||
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
||||||
import org.apache.hadoop.yarn.util.Apps;
|
import org.apache.hadoop.yarn.util.Apps;
|
||||||
|
@ -608,8 +609,7 @@ public class MRApps extends Apps {
|
||||||
}
|
}
|
||||||
String linkName = name.toUri().getPath();
|
String linkName = name.toUri().getPath();
|
||||||
LocalResource orig = localResources.get(linkName);
|
LocalResource orig = localResources.get(linkName);
|
||||||
org.apache.hadoop.yarn.api.records.URL url =
|
URL url = URL.fromURI(p.toUri());
|
||||||
ConverterUtils.getYarnUrlFromURI(p.toUri());
|
|
||||||
if(orig != null && !orig.getResource().equals(url)) {
|
if(orig != null && !orig.getResource().equals(url)) {
|
||||||
LOG.warn(
|
LOG.warn(
|
||||||
getResourceDescription(orig.getType()) +
|
getResourceDescription(orig.getType()) +
|
||||||
|
@ -618,8 +618,8 @@ public class MRApps extends Apps {
|
||||||
" This will be an error in Hadoop 2.0");
|
" This will be an error in Hadoop 2.0");
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
localResources.put(linkName, LocalResource.newInstance(ConverterUtils
|
localResources.put(linkName, LocalResource
|
||||||
.getYarnUrlFromURI(p.toUri()), type, visibilities[i]
|
.newInstance(URL.fromURI(p.toUri()), type, visibilities[i]
|
||||||
? LocalResourceVisibility.PUBLIC : LocalResourceVisibility.PRIVATE,
|
? LocalResourceVisibility.PUBLIC : LocalResourceVisibility.PRIVATE,
|
||||||
sizes[i], timestamps[i]));
|
sizes[i], timestamps[i]));
|
||||||
}
|
}
|
||||||
|
|
|
@ -107,8 +107,8 @@ public class AMStartedEvent implements HistoryEvent {
|
||||||
* @return the ApplicationAttemptId
|
* @return the ApplicationAttemptId
|
||||||
*/
|
*/
|
||||||
public ApplicationAttemptId getAppAttemptId() {
|
public ApplicationAttemptId getAppAttemptId() {
|
||||||
return ConverterUtils.toApplicationAttemptId(datum.getApplicationAttemptId()
|
return ApplicationAttemptId.fromString(
|
||||||
.toString());
|
datum.getApplicationAttemptId().toString());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -122,7 +122,7 @@ public class AMStartedEvent implements HistoryEvent {
|
||||||
* @return the ContainerId for the MRAppMaster.
|
* @return the ContainerId for the MRAppMaster.
|
||||||
*/
|
*/
|
||||||
public ContainerId getContainerId() {
|
public ContainerId getContainerId() {
|
||||||
return ConverterUtils.toContainerId(datum.getContainerId().toString());
|
return ContainerId.fromString(datum.getContainerId().toString());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -75,7 +75,7 @@ public class TaskAttemptStartedEvent implements HistoryEvent {
|
||||||
long startTime, String trackerName, int httpPort, int shufflePort,
|
long startTime, String trackerName, int httpPort, int shufflePort,
|
||||||
String locality, String avataar) {
|
String locality, String avataar) {
|
||||||
this(attemptId, taskType, startTime, trackerName, httpPort, shufflePort,
|
this(attemptId, taskType, startTime, trackerName, httpPort, shufflePort,
|
||||||
ConverterUtils.toContainerId("container_-1_-1_-1_-1"), locality,
|
ContainerId.fromString("container_-1_-1_-1_-1"), locality,
|
||||||
avataar);
|
avataar);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -116,7 +116,7 @@ public class TaskAttemptStartedEvent implements HistoryEvent {
|
||||||
}
|
}
|
||||||
/** Get the ContainerId */
|
/** Get the ContainerId */
|
||||||
public ContainerId getContainerId() {
|
public ContainerId getContainerId() {
|
||||||
return ConverterUtils.toContainerId(datum.getContainerId().toString());
|
return ContainerId.fromString(datum.getContainerId().toString());
|
||||||
}
|
}
|
||||||
/** Get the locality */
|
/** Get the locality */
|
||||||
public String getLocality() {
|
public String getLocality() {
|
||||||
|
|
|
@ -533,7 +533,7 @@ public class TestHsWebServicesAttempts extends JerseyTest {
|
||||||
WebServicesTestUtils.checkStringMatch("diagnostics", expectDiag,
|
WebServicesTestUtils.checkStringMatch("diagnostics", expectDiag,
|
||||||
diagnostics);
|
diagnostics);
|
||||||
WebServicesTestUtils.checkStringMatch("assignedContainerId",
|
WebServicesTestUtils.checkStringMatch("assignedContainerId",
|
||||||
ConverterUtils.toString(ta.getAssignedContainerID()),
|
ta.getAssignedContainerID().toString(),
|
||||||
assignedContainerId);
|
assignedContainerId);
|
||||||
|
|
||||||
assertEquals("startTime wrong", ta.getLaunchTime(), startTime);
|
assertEquals("startTime wrong", ta.getLaunchTime(), startTime);
|
||||||
|
|
|
@ -321,7 +321,7 @@ public class YARNRunner implements ClientProtocol {
|
||||||
throws IOException {
|
throws IOException {
|
||||||
LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
FileStatus rsrcStat = fs.getFileStatus(p);
|
FileStatus rsrcStat = fs.getFileStatus(p);
|
||||||
rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs
|
rsrc.setResource(URL.fromPath(fs
|
||||||
.getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
|
.getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
|
||||||
rsrc.setSize(rsrcStat.getLen());
|
rsrc.setSize(rsrcStat.getLen());
|
||||||
rsrc.setTimestamp(rsrcStat.getModificationTime());
|
rsrc.setTimestamp(rsrcStat.getModificationTime());
|
||||||
|
@ -355,8 +355,7 @@ public class YARNRunner implements ClientProtocol {
|
||||||
|
|
||||||
Path jobConfPath = new Path(jobSubmitDir, MRJobConfig.JOB_CONF_FILE);
|
Path jobConfPath = new Path(jobSubmitDir, MRJobConfig.JOB_CONF_FILE);
|
||||||
|
|
||||||
URL yarnUrlForJobSubmitDir = ConverterUtils
|
URL yarnUrlForJobSubmitDir = URL.fromPath(defaultFileContext.getDefaultFileSystem()
|
||||||
.getYarnUrlFromPath(defaultFileContext.getDefaultFileSystem()
|
|
||||||
.resolvePath(
|
.resolvePath(
|
||||||
defaultFileContext.makeQualified(new Path(jobSubmitDir))));
|
defaultFileContext.makeQualified(new Path(jobSubmitDir))));
|
||||||
LOG.debug("Creating setup context, jobSubmitDir url is "
|
LOG.debug("Creating setup context, jobSubmitDir url is "
|
||||||
|
|
|
@ -749,7 +749,7 @@ public class TestMRJobs {
|
||||||
boolean foundAppMaster = job.isUber();
|
boolean foundAppMaster = job.isUber();
|
||||||
final Path containerPathComponent = slog.getPath().getParent();
|
final Path containerPathComponent = slog.getPath().getParent();
|
||||||
if (!foundAppMaster) {
|
if (!foundAppMaster) {
|
||||||
final ContainerId cid = ConverterUtils.toContainerId(
|
final ContainerId cid = ContainerId.fromString(
|
||||||
containerPathComponent.getName());
|
containerPathComponent.getName());
|
||||||
foundAppMaster =
|
foundAppMaster =
|
||||||
((cid.getContainerId() & ContainerId.CONTAINER_ID_BITMASK)== 1);
|
((cid.getContainerId() & ContainerId.CONTAINER_ID_BITMASK)== 1);
|
||||||
|
|
|
@ -999,7 +999,7 @@ public class ShuffleHandler extends AuxiliaryService {
|
||||||
final String baseStr =
|
final String baseStr =
|
||||||
ContainerLocalizer.USERCACHE + "/" + user + "/"
|
ContainerLocalizer.USERCACHE + "/" + user + "/"
|
||||||
+ ContainerLocalizer.APPCACHE + "/"
|
+ ContainerLocalizer.APPCACHE + "/"
|
||||||
+ ConverterUtils.toString(appID) + "/output" + "/";
|
+ appID.toString() + "/output" + "/";
|
||||||
return baseStr;
|
return baseStr;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -39,6 +39,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
import org.apache.hadoop.mapred.JobConf;
|
import org.apache.hadoop.mapred.JobConf;
|
||||||
import org.apache.hadoop.util.Tool;
|
import org.apache.hadoop.util.Tool;
|
||||||
import org.apache.hadoop.util.ToolRunner;
|
import org.apache.hadoop.util.ToolRunner;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
||||||
import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
|
import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
|
||||||
import org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster;
|
import org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster;
|
||||||
|
@ -302,7 +303,7 @@ public class HadoopArchiveLogs implements Tool {
|
||||||
AppInfo app = it.next();
|
AppInfo app = it.next();
|
||||||
try {
|
try {
|
||||||
ApplicationReport report = client.getApplicationReport(
|
ApplicationReport report = client.getApplicationReport(
|
||||||
ConverterUtils.toApplicationId(app.getAppId()));
|
ApplicationId.fromString(app.getAppId()));
|
||||||
LogAggregationStatus aggStatus = report.getLogAggregationStatus();
|
LogAggregationStatus aggStatus = report.getLogAggregationStatus();
|
||||||
if (aggStatus.equals(LogAggregationStatus.RUNNING) ||
|
if (aggStatus.equals(LogAggregationStatus.RUNNING) ||
|
||||||
aggStatus.equals(LogAggregationStatus.RUNNING_WITH_FAILURE) ||
|
aggStatus.equals(LogAggregationStatus.RUNNING_WITH_FAILURE) ||
|
||||||
|
|
|
@ -19,6 +19,8 @@
|
||||||
package org.apache.hadoop.yarn.api.records;
|
package org.apache.hadoop.yarn.api.records;
|
||||||
|
|
||||||
import java.text.NumberFormat;
|
import java.text.NumberFormat;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.NoSuchElementException;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
|
@ -26,6 +28,8 @@ import org.apache.hadoop.classification.InterfaceStability.Stable;
|
||||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
|
|
||||||
|
import com.google.common.base.Splitter;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* <p><code>ApplicationAttemptId</code> denotes the particular <em>attempt</em>
|
* <p><code>ApplicationAttemptId</code> denotes the particular <em>attempt</em>
|
||||||
* of an <code>ApplicationMaster</code> for a given {@link ApplicationId}.</p>
|
* of an <code>ApplicationMaster</code> for a given {@link ApplicationId}.</p>
|
||||||
|
@ -38,10 +42,11 @@ import org.apache.hadoop.yarn.util.Records;
|
||||||
@Stable
|
@Stable
|
||||||
public abstract class ApplicationAttemptId implements
|
public abstract class ApplicationAttemptId implements
|
||||||
Comparable<ApplicationAttemptId> {
|
Comparable<ApplicationAttemptId> {
|
||||||
|
private static Splitter _spliter = Splitter.on('_').trimResults();
|
||||||
|
|
||||||
@Private
|
@Private
|
||||||
@Unstable
|
@Unstable
|
||||||
public static final String appAttemptIdStrPrefix = "appattempt_";
|
public static final String appAttemptIdStrPrefix = "appattempt";
|
||||||
|
|
||||||
@Public
|
@Public
|
||||||
@Unstable
|
@Unstable
|
||||||
|
@ -131,6 +136,7 @@ public abstract class ApplicationAttemptId implements
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
StringBuilder sb = new StringBuilder(appAttemptIdStrPrefix);
|
StringBuilder sb = new StringBuilder(appAttemptIdStrPrefix);
|
||||||
|
sb.append("_");
|
||||||
sb.append(this.getApplicationId().getClusterTimestamp()).append("_");
|
sb.append(this.getApplicationId().getClusterTimestamp()).append("_");
|
||||||
sb.append(ApplicationId.appIdFormat.get().format(
|
sb.append(ApplicationId.appIdFormat.get().format(
|
||||||
this.getApplicationId().getId()));
|
this.getApplicationId().getId()));
|
||||||
|
@ -139,4 +145,32 @@ public abstract class ApplicationAttemptId implements
|
||||||
}
|
}
|
||||||
|
|
||||||
protected abstract void build();
|
protected abstract void build();
|
||||||
|
|
||||||
|
@Public
|
||||||
|
@Stable
|
||||||
|
public static ApplicationAttemptId fromString(String applicationAttemptIdStr) {
|
||||||
|
Iterator<String> it = _spliter.split(applicationAttemptIdStr).iterator();
|
||||||
|
if (!it.next().equals(appAttemptIdStrPrefix)) {
|
||||||
|
throw new IllegalArgumentException("Invalid AppAttemptId prefix: "
|
||||||
|
+ applicationAttemptIdStr);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return toApplicationAttemptId(it);
|
||||||
|
} catch (NumberFormatException n) {
|
||||||
|
throw new IllegalArgumentException("Invalid AppAttemptId: "
|
||||||
|
+ applicationAttemptIdStr, n);
|
||||||
|
} catch (NoSuchElementException e) {
|
||||||
|
throw new IllegalArgumentException("Invalid AppAttemptId: "
|
||||||
|
+ applicationAttemptIdStr, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ApplicationAttemptId toApplicationAttemptId(
|
||||||
|
Iterator<String> it) throws NumberFormatException {
|
||||||
|
ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
|
||||||
|
Integer.parseInt(it.next()));
|
||||||
|
ApplicationAttemptId appAttemptId =
|
||||||
|
ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next()));
|
||||||
|
return appAttemptId;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,6 +19,8 @@
|
||||||
package org.apache.hadoop.yarn.api.records;
|
package org.apache.hadoop.yarn.api.records;
|
||||||
|
|
||||||
import java.text.NumberFormat;
|
import java.text.NumberFormat;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.NoSuchElementException;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
|
@ -26,6 +28,8 @@ import org.apache.hadoop.classification.InterfaceStability.Stable;
|
||||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
|
|
||||||
|
import com.google.common.base.Splitter;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* <p><code>ApplicationId</code> represents the <em>globally unique</em>
|
* <p><code>ApplicationId</code> represents the <em>globally unique</em>
|
||||||
* identifier for an application.</p>
|
* identifier for an application.</p>
|
||||||
|
@ -38,10 +42,11 @@ import org.apache.hadoop.yarn.util.Records;
|
||||||
@Public
|
@Public
|
||||||
@Stable
|
@Stable
|
||||||
public abstract class ApplicationId implements Comparable<ApplicationId> {
|
public abstract class ApplicationId implements Comparable<ApplicationId> {
|
||||||
|
private static Splitter _spliter = Splitter.on('_').trimResults();
|
||||||
|
|
||||||
@Private
|
@Private
|
||||||
@Unstable
|
@Unstable
|
||||||
public static final String appIdStrPrefix = "application_";
|
public static final String appIdStrPrefix = "application";
|
||||||
|
|
||||||
@Public
|
@Public
|
||||||
@Unstable
|
@Unstable
|
||||||
|
@ -105,8 +110,35 @@ public abstract class ApplicationId implements Comparable<ApplicationId> {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return appIdStrPrefix + this.getClusterTimestamp() + "_"
|
return appIdStrPrefix + "_" + this.getClusterTimestamp() + "_" + appIdFormat
|
||||||
+ appIdFormat.get().format(getId());
|
.get().format(getId());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ApplicationId toApplicationId(
|
||||||
|
Iterator<String> it) throws NumberFormatException {
|
||||||
|
ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
|
||||||
|
Integer.parseInt(it.next()));
|
||||||
|
return appId;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Public
|
||||||
|
@Stable
|
||||||
|
public static ApplicationId fromString(String appIdStr) {
|
||||||
|
Iterator<String> it = _spliter.split((appIdStr)).iterator();
|
||||||
|
if (!it.next().equals(appIdStrPrefix)) {
|
||||||
|
throw new IllegalArgumentException("Invalid ApplicationId prefix: "
|
||||||
|
+ appIdStr + ". The valid ApplicationId should start with prefix "
|
||||||
|
+ appIdStrPrefix);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return toApplicationId(it);
|
||||||
|
} catch (NumberFormatException n) {
|
||||||
|
throw new IllegalArgumentException("Invalid ApplicationId: "
|
||||||
|
+ appIdStr, n);
|
||||||
|
} catch (NoSuchElementException e) {
|
||||||
|
throw new IllegalArgumentException("Invalid ApplicationId: "
|
||||||
|
+ appIdStr, e);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -42,7 +42,7 @@ public abstract class ContainerId implements Comparable<ContainerId>{
|
||||||
private static final String CONTAINER_PREFIX = "container";
|
private static final String CONTAINER_PREFIX = "container";
|
||||||
private static final String EPOCH_PREFIX = "e";
|
private static final String EPOCH_PREFIX = "e";
|
||||||
|
|
||||||
@Private
|
@Public
|
||||||
@Unstable
|
@Unstable
|
||||||
public static ContainerId newContainerId(ApplicationAttemptId appAttemptId,
|
public static ContainerId newContainerId(ApplicationAttemptId appAttemptId,
|
||||||
long containerId) {
|
long containerId) {
|
||||||
|
@ -97,7 +97,7 @@ public abstract class ContainerId implements Comparable<ContainerId>{
|
||||||
*/
|
*/
|
||||||
@Public
|
@Public
|
||||||
@Deprecated
|
@Deprecated
|
||||||
@Stable
|
@Unstable
|
||||||
public abstract int getId();
|
public abstract int getId();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -205,7 +205,7 @@ public abstract class ContainerId implements Comparable<ContainerId>{
|
||||||
}
|
}
|
||||||
|
|
||||||
@Public
|
@Public
|
||||||
@Unstable
|
@Stable
|
||||||
public static ContainerId fromString(String containerIdStr) {
|
public static ContainerId fromString(String containerIdStr) {
|
||||||
Iterator<String> it = _SPLITTER.split(containerIdStr).iterator();
|
Iterator<String> it = _SPLITTER.split(containerIdStr).iterator();
|
||||||
if (!it.next().equals(CONTAINER_PREFIX)) {
|
if (!it.next().equals(CONTAINER_PREFIX)) {
|
||||||
|
|
|
@ -20,8 +20,8 @@ package org.apache.hadoop.yarn.api.records;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
import org.apache.hadoop.classification.InterfaceStability.Stable;
|
|
||||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability.Stable;
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -35,8 +35,8 @@ import org.apache.hadoop.yarn.util.Records;
|
||||||
@Stable
|
@Stable
|
||||||
public abstract class NodeId implements Comparable<NodeId> {
|
public abstract class NodeId implements Comparable<NodeId> {
|
||||||
|
|
||||||
@Private
|
@Public
|
||||||
@Unstable
|
@Stable
|
||||||
public static NodeId newInstance(String host, int port) {
|
public static NodeId newInstance(String host, int port) {
|
||||||
NodeId nodeId = Records.newRecord(NodeId.class);
|
NodeId nodeId = Records.newRecord(NodeId.class);
|
||||||
nodeId.setHost(host);
|
nodeId.setHost(host);
|
||||||
|
@ -113,5 +113,22 @@ public abstract class NodeId implements Comparable<NodeId> {
|
||||||
return hostCompare;
|
return hostCompare;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Public
|
||||||
|
@Stable
|
||||||
|
public static NodeId fromString(String nodeIdStr) {
|
||||||
|
String[] parts = nodeIdStr.split(":");
|
||||||
|
if (parts.length != 2) {
|
||||||
|
throw new IllegalArgumentException("Invalid NodeId [" + nodeIdStr
|
||||||
|
+ "]. Expected host:port");
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
NodeId nodeId =
|
||||||
|
NodeId.newInstance(parts[0].trim(), Integer.parseInt(parts[1]));
|
||||||
|
return nodeId;
|
||||||
|
} catch (NumberFormatException e) {
|
||||||
|
throw new IllegalArgumentException("Invalid port: " + parts[1], e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
protected abstract void build();
|
protected abstract void build();
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,8 +18,13 @@
|
||||||
|
|
||||||
package org.apache.hadoop.yarn.api.records;
|
package org.apache.hadoop.yarn.api.records;
|
||||||
|
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
import org.apache.hadoop.classification.InterfaceStability.Stable;
|
import org.apache.hadoop.classification.InterfaceStability.Stable;
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -119,4 +124,48 @@ public abstract class URL {
|
||||||
@Public
|
@Public
|
||||||
@Stable
|
@Stable
|
||||||
public abstract void setFile(String file);
|
public abstract void setFile(String file);
|
||||||
|
|
||||||
|
@Public
|
||||||
|
@Stable
|
||||||
|
public Path toPath() throws URISyntaxException {
|
||||||
|
String scheme = getScheme() == null ? "" : getScheme();
|
||||||
|
|
||||||
|
String authority = "";
|
||||||
|
if (getHost() != null) {
|
||||||
|
authority = getHost();
|
||||||
|
if (getUserInfo() != null) {
|
||||||
|
authority = getUserInfo() + "@" + authority;
|
||||||
|
}
|
||||||
|
if (getPort() > 0) {
|
||||||
|
authority += ":" + getPort();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return new Path(
|
||||||
|
(new URI(scheme, authority, getFile(), null, null)).normalize());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Public
|
||||||
|
@Stable
|
||||||
|
public static URL fromURI(URI uri) {
|
||||||
|
URL url =
|
||||||
|
RecordFactoryProvider.getRecordFactory(null).newRecordInstance(
|
||||||
|
URL.class);
|
||||||
|
if (uri.getHost() != null) {
|
||||||
|
url.setHost(uri.getHost());
|
||||||
|
}
|
||||||
|
if (uri.getUserInfo() != null) {
|
||||||
|
url.setUserInfo(uri.getUserInfo());
|
||||||
|
}
|
||||||
|
url.setPort(uri.getPort());
|
||||||
|
url.setScheme(uri.getScheme());
|
||||||
|
url.setFile(uri.getPath());
|
||||||
|
return url;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Public
|
||||||
|
@Stable
|
||||||
|
public static URL fromPath(Path path) {
|
||||||
|
return fromURI(path.toUri());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -435,13 +435,13 @@ public class ApplicationMaster {
|
||||||
if (!envs.containsKey(Environment.CONTAINER_ID.name())) {
|
if (!envs.containsKey(Environment.CONTAINER_ID.name())) {
|
||||||
if (cliParser.hasOption("app_attempt_id")) {
|
if (cliParser.hasOption("app_attempt_id")) {
|
||||||
String appIdStr = cliParser.getOptionValue("app_attempt_id", "");
|
String appIdStr = cliParser.getOptionValue("app_attempt_id", "");
|
||||||
appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
|
appAttemptID = ApplicationAttemptId.fromString(appIdStr);
|
||||||
} else {
|
} else {
|
||||||
throw new IllegalArgumentException(
|
throw new IllegalArgumentException(
|
||||||
"Application Attempt Id not set in the environment");
|
"Application Attempt Id not set in the environment");
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(envs
|
ContainerId containerId = ContainerId.fromString(envs
|
||||||
.get(Environment.CONTAINER_ID.name()));
|
.get(Environment.CONTAINER_ID.name()));
|
||||||
appAttemptID = containerId.getApplicationAttemptId();
|
appAttemptID = containerId.getApplicationAttemptId();
|
||||||
}
|
}
|
||||||
|
@ -1048,8 +1048,7 @@ public class ApplicationMaster {
|
||||||
|
|
||||||
URL yarnUrl = null;
|
URL yarnUrl = null;
|
||||||
try {
|
try {
|
||||||
yarnUrl = ConverterUtils.getYarnUrlFromURI(
|
yarnUrl = URL.fromURI(new URI(renamedScriptPath.toString()));
|
||||||
new URI(renamedScriptPath.toString()));
|
|
||||||
} catch (URISyntaxException e) {
|
} catch (URISyntaxException e) {
|
||||||
LOG.error("Error when trying to use shell script path specified"
|
LOG.error("Error when trying to use shell script path specified"
|
||||||
+ " in env, path=" + renamedScriptPath, e);
|
+ " in env, path=" + renamedScriptPath, e);
|
||||||
|
|
|
@ -68,6 +68,7 @@ import org.apache.hadoop.yarn.api.records.QueueACL;
|
||||||
import org.apache.hadoop.yarn.api.records.QueueInfo;
|
import org.apache.hadoop.yarn.api.records.QueueInfo;
|
||||||
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
|
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
|
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
|
||||||
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
|
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
|
||||||
|
@ -857,7 +858,7 @@ public class Client {
|
||||||
FileStatus scFileStatus = fs.getFileStatus(dst);
|
FileStatus scFileStatus = fs.getFileStatus(dst);
|
||||||
LocalResource scRsrc =
|
LocalResource scRsrc =
|
||||||
LocalResource.newInstance(
|
LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromURI(dst.toUri()),
|
URL.fromURI(dst.toUri()),
|
||||||
LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
|
LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
|
||||||
scFileStatus.getLen(), scFileStatus.getModificationTime());
|
scFileStatus.getLen(), scFileStatus.getModificationTime());
|
||||||
localResources.put(fileDstPath, scRsrc);
|
localResources.put(fileDstPath, scRsrc);
|
||||||
|
|
|
@ -53,7 +53,7 @@ public class DistributedShellTimelinePlugin extends TimelineEntityGroupPlugin {
|
||||||
public Set<TimelineEntityGroupId> getTimelineEntityGroupId(String entityId,
|
public Set<TimelineEntityGroupId> getTimelineEntityGroupId(String entityId,
|
||||||
String entityType) {
|
String entityType) {
|
||||||
if (ApplicationMaster.DSEntity.DS_CONTAINER.toString().equals(entityId)) {
|
if (ApplicationMaster.DSEntity.DS_CONTAINER.toString().equals(entityId)) {
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(entityId);
|
ContainerId containerId = ContainerId.fromString(entityId);
|
||||||
ApplicationId appId = containerId.getApplicationAttemptId()
|
ApplicationId appId = containerId.getApplicationAttemptId()
|
||||||
.getApplicationId();
|
.getApplicationId();
|
||||||
return toEntityGroupId(appId.toString());
|
return toEntityGroupId(appId.toString());
|
||||||
|
@ -69,7 +69,7 @@ public class DistributedShellTimelinePlugin extends TimelineEntityGroupPlugin {
|
||||||
}
|
}
|
||||||
|
|
||||||
private Set<TimelineEntityGroupId> toEntityGroupId(String strAppId) {
|
private Set<TimelineEntityGroupId> toEntityGroupId(String strAppId) {
|
||||||
ApplicationId appId = ConverterUtils.toApplicationId(strAppId);
|
ApplicationId appId = ApplicationId.fromString(strAppId);
|
||||||
TimelineEntityGroupId groupId = TimelineEntityGroupId.newInstance(
|
TimelineEntityGroupId groupId = TimelineEntityGroupId.newInstance(
|
||||||
appId, ApplicationMaster.CONTAINER_ENTITY_GROUP_ID);
|
appId, ApplicationMaster.CONTAINER_ENTITY_GROUP_ID);
|
||||||
Set<TimelineEntityGroupId> result = new HashSet<>();
|
Set<TimelineEntityGroupId> result = new HashSet<>();
|
||||||
|
|
|
@ -371,8 +371,8 @@ public class TestDistributedShell {
|
||||||
}
|
}
|
||||||
String currAttemptEntityId
|
String currAttemptEntityId
|
||||||
= entitiesAttempts.getEntities().get(0).getEntityId();
|
= entitiesAttempts.getEntities().get(0).getEntityId();
|
||||||
ApplicationAttemptId attemptId
|
ApplicationAttemptId attemptId = ApplicationAttemptId.fromString(
|
||||||
= ConverterUtils.toApplicationAttemptId(currAttemptEntityId);
|
currAttemptEntityId);
|
||||||
NameValuePair primaryFilter = new NameValuePair(
|
NameValuePair primaryFilter = new NameValuePair(
|
||||||
ApplicationMaster.APPID_TIMELINE_FILTER_NAME,
|
ApplicationMaster.APPID_TIMELINE_FILTER_NAME,
|
||||||
attemptId.getApplicationId().toString());
|
attemptId.getApplicationId().toString());
|
||||||
|
|
|
@ -301,7 +301,7 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
*/
|
*/
|
||||||
private void signalToContainer(String containerIdStr,
|
private void signalToContainer(String containerIdStr,
|
||||||
SignalContainerCommand command) throws YarnException, IOException {
|
SignalContainerCommand command) throws YarnException, IOException {
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
sysout.println("Signalling container " + containerIdStr);
|
sysout.println("Signalling container " + containerIdStr);
|
||||||
client.signalToContainer(containerId, command);
|
client.signalToContainer(containerId, command);
|
||||||
}
|
}
|
||||||
|
@ -327,8 +327,8 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
throws YarnException, IOException {
|
throws YarnException, IOException {
|
||||||
ApplicationAttemptReport appAttemptReport = null;
|
ApplicationAttemptReport appAttemptReport = null;
|
||||||
try {
|
try {
|
||||||
appAttemptReport = client.getApplicationAttemptReport(ConverterUtils
|
appAttemptReport = client.getApplicationAttemptReport(
|
||||||
.toApplicationAttemptId(applicationAttemptId));
|
ApplicationAttemptId.fromString(applicationAttemptId));
|
||||||
} catch (ApplicationNotFoundException e) {
|
} catch (ApplicationNotFoundException e) {
|
||||||
sysout.println("Application for AppAttempt with id '"
|
sysout.println("Application for AppAttempt with id '"
|
||||||
+ applicationAttemptId + "' doesn't exist in RM or Timeline Server.");
|
+ applicationAttemptId + "' doesn't exist in RM or Timeline Server.");
|
||||||
|
@ -384,8 +384,7 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
IOException {
|
IOException {
|
||||||
ContainerReport containerReport = null;
|
ContainerReport containerReport = null;
|
||||||
try {
|
try {
|
||||||
containerReport = client.getContainerReport((ConverterUtils
|
containerReport = client.getContainerReport(ContainerId.fromString(containerId));
|
||||||
.toContainerId(containerId)));
|
|
||||||
} catch (ApplicationNotFoundException e) {
|
} catch (ApplicationNotFoundException e) {
|
||||||
sysout.println("Application for Container with id '" + containerId
|
sysout.println("Application for Container with id '" + containerId
|
||||||
+ "' doesn't exist in RM or Timeline Server.");
|
+ "' doesn't exist in RM or Timeline Server.");
|
||||||
|
@ -515,7 +514,7 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
*/
|
*/
|
||||||
private void killApplication(String applicationId) throws YarnException,
|
private void killApplication(String applicationId) throws YarnException,
|
||||||
IOException {
|
IOException {
|
||||||
ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
|
ApplicationId appId = ApplicationId.fromString(applicationId);
|
||||||
ApplicationReport appReport = null;
|
ApplicationReport appReport = null;
|
||||||
try {
|
try {
|
||||||
appReport = client.getApplicationReport(appId);
|
appReport = client.getApplicationReport(appId);
|
||||||
|
@ -540,7 +539,7 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
*/
|
*/
|
||||||
private void moveApplicationAcrossQueues(String applicationId, String queue)
|
private void moveApplicationAcrossQueues(String applicationId, String queue)
|
||||||
throws YarnException, IOException {
|
throws YarnException, IOException {
|
||||||
ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
|
ApplicationId appId = ApplicationId.fromString(applicationId);
|
||||||
ApplicationReport appReport = client.getApplicationReport(appId);
|
ApplicationReport appReport = client.getApplicationReport(appId);
|
||||||
if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
|
if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
|
||||||
|| appReport.getYarnApplicationState() == YarnApplicationState.KILLED
|
|| appReport.getYarnApplicationState() == YarnApplicationState.KILLED
|
||||||
|
@ -565,7 +564,7 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
IOException {
|
IOException {
|
||||||
ApplicationId appId;
|
ApplicationId appId;
|
||||||
ApplicationAttemptId attId;
|
ApplicationAttemptId attId;
|
||||||
attId = ConverterUtils.toApplicationAttemptId(attemptId);
|
attId = ApplicationAttemptId.fromString(attemptId);
|
||||||
appId = attId.getApplicationId();
|
appId = attId.getApplicationId();
|
||||||
|
|
||||||
sysout.println("Failing attempt " + attId + " of application " + appId);
|
sysout.println("Failing attempt " + attId + " of application " + appId);
|
||||||
|
@ -583,8 +582,8 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
throws YarnException, IOException {
|
throws YarnException, IOException {
|
||||||
ApplicationReport appReport = null;
|
ApplicationReport appReport = null;
|
||||||
try {
|
try {
|
||||||
appReport = client.getApplicationReport(ConverterUtils
|
appReport = client.getApplicationReport(
|
||||||
.toApplicationId(applicationId));
|
ApplicationId.fromString(applicationId));
|
||||||
} catch (ApplicationNotFoundException e) {
|
} catch (ApplicationNotFoundException e) {
|
||||||
sysout.println("Application with id '" + applicationId
|
sysout.println("Application with id '" + applicationId
|
||||||
+ "' doesn't exist in RM or Timeline Server.");
|
+ "' doesn't exist in RM or Timeline Server.");
|
||||||
|
@ -684,7 +683,7 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
|
new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
|
||||||
|
|
||||||
List<ApplicationAttemptReport> appAttemptsReport = client
|
List<ApplicationAttemptReport> appAttemptsReport = client
|
||||||
.getApplicationAttempts(ConverterUtils.toApplicationId(applicationId));
|
.getApplicationAttempts(ApplicationId.fromString(applicationId));
|
||||||
writer.println("Total number of application attempts " + ":"
|
writer.println("Total number of application attempts " + ":"
|
||||||
+ appAttemptsReport.size());
|
+ appAttemptsReport.size());
|
||||||
writer.printf(APPLICATION_ATTEMPTS_PATTERN, "ApplicationAttempt-Id",
|
writer.printf(APPLICATION_ATTEMPTS_PATTERN, "ApplicationAttempt-Id",
|
||||||
|
@ -711,8 +710,8 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
PrintWriter writer = new PrintWriter(
|
PrintWriter writer = new PrintWriter(
|
||||||
new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
|
new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
|
||||||
|
|
||||||
List<ContainerReport> appsReport = client
|
List<ContainerReport> appsReport = client.getContainers(
|
||||||
.getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId));
|
ApplicationAttemptId.fromString(appAttemptId));
|
||||||
writer.println("Total number of containers " + ":" + appsReport.size());
|
writer.println("Total number of containers " + ":" + appsReport.size());
|
||||||
writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
|
writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
|
||||||
"Finish Time", "State", "Host", "Node Http Address", "LOG-URL");
|
"Finish Time", "State", "Host", "Node Http Address", "LOG-URL");
|
||||||
|
@ -735,7 +734,7 @@ public class ApplicationCLI extends YarnCLI {
|
||||||
*/
|
*/
|
||||||
private void updateApplicationPriority(String applicationId, String priority)
|
private void updateApplicationPriority(String applicationId, String priority)
|
||||||
throws YarnException, IOException {
|
throws YarnException, IOException {
|
||||||
ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
|
ApplicationId appId = ApplicationId.fromString(applicationId);
|
||||||
Priority newAppPriority = Priority.newInstance(Integer.parseInt(priority));
|
Priority newAppPriority = Priority.newInstance(Integer.parseInt(priority));
|
||||||
sysout.println("Updating priority of an application " + applicationId);
|
sysout.println("Updating priority of an application " + applicationId);
|
||||||
Priority updateApplicationPriority =
|
Priority updateApplicationPriority =
|
||||||
|
|
|
@ -153,7 +153,7 @@ public class LogsCLI extends Configured implements Tool {
|
||||||
|
|
||||||
ApplicationId appId = null;
|
ApplicationId appId = null;
|
||||||
try {
|
try {
|
||||||
appId = ConverterUtils.toApplicationId(appIdStr);
|
appId = ApplicationId.fromString(appIdStr);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
System.err.println("Invalid ApplicationId specified");
|
System.err.println("Invalid ApplicationId specified");
|
||||||
return -1;
|
return -1;
|
||||||
|
@ -456,8 +456,8 @@ public class LogsCLI extends Configured implements Tool {
|
||||||
throws YarnException, IOException {
|
throws YarnException, IOException {
|
||||||
YarnClient yarnClient = createYarnClient();
|
YarnClient yarnClient = createYarnClient();
|
||||||
try {
|
try {
|
||||||
return yarnClient.getContainerReport(ConverterUtils
|
return yarnClient.getContainerReport(
|
||||||
.toContainerId(containerIdStr));
|
ContainerId.fromString(containerIdStr));
|
||||||
} finally {
|
} finally {
|
||||||
yarnClient.close();
|
yarnClient.close();
|
||||||
}
|
}
|
||||||
|
|
|
@ -243,7 +243,7 @@ public class NodeCLI extends YarnCLI {
|
||||||
*/
|
*/
|
||||||
private void printNodeStatus(String nodeIdStr) throws YarnException,
|
private void printNodeStatus(String nodeIdStr) throws YarnException,
|
||||||
IOException {
|
IOException {
|
||||||
NodeId nodeId = ConverterUtils.toNodeId(nodeIdStr);
|
NodeId nodeId = NodeId.fromString(nodeIdStr);
|
||||||
List<NodeReport> nodesReport = client.getNodeReports();
|
List<NodeReport> nodesReport = client.getNodeReports();
|
||||||
// Use PrintWriter.println, which uses correct platform line ending.
|
// Use PrintWriter.println, which uses correct platform line ending.
|
||||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||||
|
|
|
@ -427,7 +427,7 @@ public class RMAdminCLI extends HAAdmin {
|
||||||
ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol();
|
ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol();
|
||||||
UpdateNodeResourceRequest request =
|
UpdateNodeResourceRequest request =
|
||||||
recordFactory.newRecordInstance(UpdateNodeResourceRequest.class);
|
recordFactory.newRecordInstance(UpdateNodeResourceRequest.class);
|
||||||
NodeId nodeId = ConverterUtils.toNodeId(nodeIdStr);
|
NodeId nodeId = NodeId.fromString(nodeIdStr);
|
||||||
|
|
||||||
Resource resource = Resources.createResource(memSize, cores);
|
Resource resource = Resources.createResource(memSize, cores);
|
||||||
Map<NodeId, ResourceOption> resourceMap =
|
Map<NodeId, ResourceOption> resourceMap =
|
||||||
|
|
|
@ -222,7 +222,7 @@ public class TestRMAdminCLI {
|
||||||
verify(admin).updateNodeResource(argument.capture());
|
verify(admin).updateNodeResource(argument.capture());
|
||||||
UpdateNodeResourceRequest request = argument.getValue();
|
UpdateNodeResourceRequest request = argument.getValue();
|
||||||
Map<NodeId, ResourceOption> resourceMap = request.getNodeResourceMap();
|
Map<NodeId, ResourceOption> resourceMap = request.getNodeResourceMap();
|
||||||
NodeId nodeId = ConverterUtils.toNodeId(nodeIdStr);
|
NodeId nodeId = NodeId.fromString(nodeIdStr);
|
||||||
Resource expectedResource = Resources.createResource(memSize, cores);
|
Resource expectedResource = Resources.createResource(memSize, cores);
|
||||||
ResourceOption resource = resourceMap.get(nodeId);
|
ResourceOption resource = resourceMap.get(nodeId);
|
||||||
assertNotNull("resource for " + nodeIdStr + " shouldn't be null.",
|
assertNotNull("resource for " + nodeIdStr + " shouldn't be null.",
|
||||||
|
|
|
@ -99,7 +99,7 @@ public class AggregatedLogDeletionService extends AbstractService {
|
||||||
if(appDir.isDirectory() &&
|
if(appDir.isDirectory() &&
|
||||||
appDir.getModificationTime() < cutoffMillis) {
|
appDir.getModificationTime() < cutoffMillis) {
|
||||||
boolean appTerminated =
|
boolean appTerminated =
|
||||||
isApplicationTerminated(ConverterUtils.toApplicationId(appDir
|
isApplicationTerminated(ApplicationId.fromString(appDir
|
||||||
.getPath().getName()), rmClient);
|
.getPath().getName()), rmClient);
|
||||||
if(appTerminated && shouldDeleteLogDir(appDir, cutoffMillis, fs)) {
|
if(appTerminated && shouldDeleteLogDir(appDir, cutoffMillis, fs)) {
|
||||||
try {
|
try {
|
||||||
|
|
|
@ -209,14 +209,11 @@ public class AggregatedLogFormat {
|
||||||
public Set<File> getPendingLogFilesToUploadForThisContainer() {
|
public Set<File> getPendingLogFilesToUploadForThisContainer() {
|
||||||
Set<File> pendingUploadFiles = new HashSet<File>();
|
Set<File> pendingUploadFiles = new HashSet<File>();
|
||||||
for (String rootLogDir : this.rootLogDirs) {
|
for (String rootLogDir : this.rootLogDirs) {
|
||||||
File appLogDir =
|
File appLogDir = new File(rootLogDir,
|
||||||
new File(rootLogDir,
|
|
||||||
ConverterUtils.toString(
|
|
||||||
this.containerId.getApplicationAttemptId().
|
this.containerId.getApplicationAttemptId().
|
||||||
getApplicationId())
|
getApplicationId().toString());
|
||||||
);
|
|
||||||
File containerLogDir =
|
File containerLogDir =
|
||||||
new File(appLogDir, ConverterUtils.toString(this.containerId));
|
new File(appLogDir, this.containerId.toString());
|
||||||
|
|
||||||
if (!containerLogDir.isDirectory()) {
|
if (!containerLogDir.isDirectory()) {
|
||||||
continue; // ContainerDir may have been deleted by the user.
|
continue; // ContainerDir may have been deleted by the user.
|
||||||
|
|
|
@ -59,7 +59,7 @@ public class LogCLIHelpers implements Configurable {
|
||||||
public int dumpAContainersLogs(String appId, String containerId,
|
public int dumpAContainersLogs(String appId, String containerId,
|
||||||
String nodeId, String jobOwner) throws IOException {
|
String nodeId, String jobOwner) throws IOException {
|
||||||
ContainerLogsRequest options = new ContainerLogsRequest();
|
ContainerLogsRequest options = new ContainerLogsRequest();
|
||||||
options.setAppId(ConverterUtils.toApplicationId(appId));
|
options.setAppId(ApplicationId.fromString(appId));
|
||||||
options.setContainerId(containerId);
|
options.setContainerId(containerId);
|
||||||
options.setNodeId(nodeId);
|
options.setNodeId(nodeId);
|
||||||
options.setAppOwner(jobOwner);
|
options.setAppOwner(jobOwner);
|
||||||
|
|
|
@ -18,18 +18,13 @@
|
||||||
|
|
||||||
package org.apache.hadoop.yarn.util;
|
package org.apache.hadoop.yarn.util;
|
||||||
|
|
||||||
import static org.apache.hadoop.yarn.util.StringHelper._split;
|
|
||||||
|
|
||||||
import java.net.InetSocketAddress;
|
import java.net.InetSocketAddress;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.URISyntaxException;
|
import java.net.URISyntaxException;
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Iterator;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Map.Entry;
|
|
||||||
import java.util.NoSuchElementException;
|
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.io.Text;
|
import org.apache.hadoop.io.Text;
|
||||||
import org.apache.hadoop.security.SecurityUtil;
|
import org.apache.hadoop.security.SecurityUtil;
|
||||||
|
@ -41,7 +36,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.api.records.URL;
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -49,7 +43,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
* from/to 'serializableFormat' to/from hadoop/nativejava data structures.
|
* from/to 'serializableFormat' to/from hadoop/nativejava data structures.
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
@Private
|
@Public
|
||||||
public class ConverterUtils {
|
public class ConverterUtils {
|
||||||
|
|
||||||
public static final String APPLICATION_PREFIX = "application";
|
public static final String APPLICATION_PREFIX = "application";
|
||||||
|
@ -58,174 +52,114 @@ public class ConverterUtils {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* return a hadoop path from a given url
|
* return a hadoop path from a given url
|
||||||
|
* This method is deprecated, use {@link URL#toPath()} instead.
|
||||||
*
|
*
|
||||||
* @param url
|
* @param url
|
||||||
* url to convert
|
* url to convert
|
||||||
* @return path from {@link URL}
|
* @return path from {@link URL}
|
||||||
* @throws URISyntaxException
|
* @throws URISyntaxException
|
||||||
*/
|
*/
|
||||||
|
@Public
|
||||||
|
@Deprecated
|
||||||
public static Path getPathFromYarnURL(URL url) throws URISyntaxException {
|
public static Path getPathFromYarnURL(URL url) throws URISyntaxException {
|
||||||
String scheme = url.getScheme() == null ? "" : url.getScheme();
|
return url.toPath();
|
||||||
|
|
||||||
String authority = "";
|
|
||||||
if (url.getHost() != null) {
|
|
||||||
authority = url.getHost();
|
|
||||||
if (url.getUserInfo() != null) {
|
|
||||||
authority = url.getUserInfo() + "@" + authority;
|
|
||||||
}
|
|
||||||
if (url.getPort() > 0) {
|
|
||||||
authority += ":" + url.getPort();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return new Path(
|
/*
|
||||||
(new URI(scheme, authority, url.getFile(), null, null)).normalize());
|
* This method is deprecated, use {@link URL#fromPath(Path)} instead.
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* change from CharSequence to string for map key and value
|
|
||||||
* @param env map for converting
|
|
||||||
* @return string,string map
|
|
||||||
*/
|
*/
|
||||||
public static Map<String, String> convertToString(
|
@Public
|
||||||
Map<CharSequence, CharSequence> env) {
|
@Deprecated
|
||||||
|
|
||||||
Map<String, String> stringMap = new HashMap<String, String>();
|
|
||||||
for (Entry<CharSequence, CharSequence> entry: env.entrySet()) {
|
|
||||||
stringMap.put(entry.getKey().toString(), entry.getValue().toString());
|
|
||||||
}
|
|
||||||
return stringMap;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static URL getYarnUrlFromPath(Path path) {
|
public static URL getYarnUrlFromPath(Path path) {
|
||||||
return getYarnUrlFromURI(path.toUri());
|
return URL.fromPath(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This method is deprecated, use {@link URL#fromURI(URI)} instead.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Deprecated
|
||||||
public static URL getYarnUrlFromURI(URI uri) {
|
public static URL getYarnUrlFromURI(URI uri) {
|
||||||
URL url = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(URL.class);
|
return URL.fromURI(uri);
|
||||||
if (uri.getHost() != null) {
|
|
||||||
url.setHost(uri.getHost());
|
|
||||||
}
|
|
||||||
if (uri.getUserInfo() != null) {
|
|
||||||
url.setUserInfo(uri.getUserInfo());
|
|
||||||
}
|
|
||||||
url.setPort(uri.getPort());
|
|
||||||
url.setScheme(uri.getScheme());
|
|
||||||
url.setFile(uri.getPath());
|
|
||||||
return url;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This method is deprecated, use {@link ApplicationId#toString()} instead.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Deprecated
|
||||||
public static String toString(ApplicationId appId) {
|
public static String toString(ApplicationId appId) {
|
||||||
return appId.toString();
|
return appId.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This method is deprecated, use {@link ApplicationId#fromString(String)}
|
||||||
|
* instead.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Deprecated
|
||||||
public static ApplicationId toApplicationId(RecordFactory recordFactory,
|
public static ApplicationId toApplicationId(RecordFactory recordFactory,
|
||||||
String appIdStr) {
|
String applicationIdStr) {
|
||||||
Iterator<String> it = _split(appIdStr).iterator();
|
return ApplicationId.fromString(applicationIdStr);
|
||||||
if (!it.next().equals(APPLICATION_PREFIX)) {
|
|
||||||
throw new IllegalArgumentException("Invalid ApplicationId prefix: "
|
|
||||||
+ appIdStr + ". The valid ApplicationId should start with prefix "
|
|
||||||
+ APPLICATION_PREFIX);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return toApplicationId(recordFactory, it);
|
|
||||||
} catch (NumberFormatException n) {
|
|
||||||
throw new IllegalArgumentException("Invalid ApplicationId: " + appIdStr,
|
|
||||||
n);
|
|
||||||
} catch (NoSuchElementException e) {
|
|
||||||
throw new IllegalArgumentException("Invalid ApplicationId: " + appIdStr,
|
|
||||||
e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static ApplicationId toApplicationId(RecordFactory recordFactory,
|
|
||||||
Iterator<String> it) {
|
|
||||||
ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
|
|
||||||
Integer.parseInt(it.next()));
|
|
||||||
return appId;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static ApplicationAttemptId toApplicationAttemptId(
|
|
||||||
Iterator<String> it) throws NumberFormatException {
|
|
||||||
ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
|
|
||||||
Integer.parseInt(it.next()));
|
|
||||||
ApplicationAttemptId appAttemptId =
|
|
||||||
ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next()));
|
|
||||||
return appAttemptId;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static ApplicationId toApplicationId(
|
|
||||||
Iterator<String> it) throws NumberFormatException {
|
|
||||||
ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
|
|
||||||
Integer.parseInt(it.next()));
|
|
||||||
return appId;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This method is deprecated, use {@link ContainerId#toString()} instead.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Deprecated
|
||||||
public static String toString(ContainerId cId) {
|
public static String toString(ContainerId cId) {
|
||||||
return cId == null ? null : cId.toString();
|
return cId == null ? null : cId.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Private
|
||||||
|
@InterfaceStability.Unstable
|
||||||
public static NodeId toNodeIdWithDefaultPort(String nodeIdStr) {
|
public static NodeId toNodeIdWithDefaultPort(String nodeIdStr) {
|
||||||
if (nodeIdStr.indexOf(":") < 0) {
|
if (nodeIdStr.indexOf(":") < 0) {
|
||||||
return toNodeId(nodeIdStr + ":0");
|
return NodeId.fromString(nodeIdStr + ":0");
|
||||||
}
|
}
|
||||||
return toNodeId(nodeIdStr);
|
return NodeId.fromString(nodeIdStr);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This method is deprecated, use {@link NodeId#fromString(String)} instead.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Deprecated
|
||||||
public static NodeId toNodeId(String nodeIdStr) {
|
public static NodeId toNodeId(String nodeIdStr) {
|
||||||
String[] parts = nodeIdStr.split(":");
|
return NodeId.fromString(nodeIdStr);
|
||||||
if (parts.length != 2) {
|
|
||||||
throw new IllegalArgumentException("Invalid NodeId [" + nodeIdStr
|
|
||||||
+ "]. Expected host:port");
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
NodeId nodeId =
|
|
||||||
NodeId.newInstance(parts[0].trim(), Integer.parseInt(parts[1]));
|
|
||||||
return nodeId;
|
|
||||||
} catch (NumberFormatException e) {
|
|
||||||
throw new IllegalArgumentException("Invalid port: " + parts[1], e);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This method is deprecated, use {@link ContainerId#fromString(String)}
|
||||||
|
* instead.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Deprecated
|
||||||
public static ContainerId toContainerId(String containerIdStr) {
|
public static ContainerId toContainerId(String containerIdStr) {
|
||||||
return ContainerId.fromString(containerIdStr);
|
return ContainerId.fromString(containerIdStr);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This method is deprecated, use {@link ApplicationAttemptId#toString()}
|
||||||
|
* instead.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Deprecated
|
||||||
public static ApplicationAttemptId toApplicationAttemptId(
|
public static ApplicationAttemptId toApplicationAttemptId(
|
||||||
String applicationAttmeptIdStr) {
|
String applicationAttemptIdStr) {
|
||||||
Iterator<String> it = _split(applicationAttmeptIdStr).iterator();
|
return ApplicationAttemptId.fromString(applicationAttemptIdStr);
|
||||||
if (!it.next().equals(APPLICATION_ATTEMPT_PREFIX)) {
|
|
||||||
throw new IllegalArgumentException("Invalid AppAttemptId prefix: "
|
|
||||||
+ applicationAttmeptIdStr);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return toApplicationAttemptId(it);
|
|
||||||
} catch (NumberFormatException n) {
|
|
||||||
throw new IllegalArgumentException("Invalid AppAttemptId: "
|
|
||||||
+ applicationAttmeptIdStr, n);
|
|
||||||
} catch (NoSuchElementException e) {
|
|
||||||
throw new IllegalArgumentException("Invalid AppAttemptId: "
|
|
||||||
+ applicationAttmeptIdStr, e);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This method is deprecated, use {@link ApplicationId#fromString(String)}
|
||||||
|
* instead.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Deprecated
|
||||||
public static ApplicationId toApplicationId(
|
public static ApplicationId toApplicationId(
|
||||||
String appIdStr) {
|
String appIdStr) {
|
||||||
Iterator<String> it = _split(appIdStr).iterator();
|
return ApplicationId.fromString(appIdStr);
|
||||||
if (!it.next().equals(APPLICATION_PREFIX)) {
|
|
||||||
throw new IllegalArgumentException("Invalid ApplicationId prefix: "
|
|
||||||
+ appIdStr + ". The valid ApplicationId should start with prefix "
|
|
||||||
+ APPLICATION_PREFIX);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return toApplicationId(it);
|
|
||||||
} catch (NumberFormatException n) {
|
|
||||||
throw new IllegalArgumentException("Invalid ApplicationId: "
|
|
||||||
+ appIdStr, n);
|
|
||||||
} catch (NoSuchElementException e) {
|
|
||||||
throw new IllegalArgumentException("Invalid ApplicationId: "
|
|
||||||
+ appIdStr, e);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -346,7 +346,7 @@ public class FSDownload implements Callable<Path> {
|
||||||
public Path call() throws Exception {
|
public Path call() throws Exception {
|
||||||
final Path sCopy;
|
final Path sCopy;
|
||||||
try {
|
try {
|
||||||
sCopy = ConverterUtils.getPathFromYarnURL(resource.getResource());
|
sCopy = resource.getResource().toPath();
|
||||||
} catch (URISyntaxException e) {
|
} catch (URISyntaxException e) {
|
||||||
throw new IOException("Invalid resource", e);
|
throw new IOException("Invalid resource", e);
|
||||||
}
|
}
|
||||||
|
|
|
@ -290,7 +290,7 @@ public class AggregatedLogsBlock extends HtmlBlock {
|
||||||
}
|
}
|
||||||
ContainerId containerId = null;
|
ContainerId containerId = null;
|
||||||
try {
|
try {
|
||||||
containerId = ConverterUtils.toContainerId(containerIdStr);
|
containerId = ContainerId.fromString(containerIdStr);
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
html.h1()
|
html.h1()
|
||||||
._("Cannot get container logs for invalid containerId: "
|
._("Cannot get container logs for invalid containerId: "
|
||||||
|
@ -308,7 +308,7 @@ public class AggregatedLogsBlock extends HtmlBlock {
|
||||||
}
|
}
|
||||||
NodeId nodeId = null;
|
NodeId nodeId = null;
|
||||||
try {
|
try {
|
||||||
nodeId = ConverterUtils.toNodeId(nodeIdStr);
|
nodeId = NodeId.fromString(nodeIdStr);
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
html.h1()._("Cannot get container logs. Invalid nodeId: " + nodeIdStr)
|
html.h1()._("Cannot get container logs. Invalid nodeId: " + nodeIdStr)
|
||||||
._();
|
._();
|
||||||
|
|
|
@ -392,7 +392,7 @@ public class WebAppUtils {
|
||||||
}
|
}
|
||||||
ApplicationId aid = null;
|
ApplicationId aid = null;
|
||||||
try {
|
try {
|
||||||
aid = ConverterUtils.toApplicationId(recordFactory, appId);
|
aid = ApplicationId.fromString(appId);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new BadRequestException(e);
|
throw new BadRequestException(e);
|
||||||
}
|
}
|
||||||
|
|
|
@ -34,55 +34,56 @@ public class TestConverterUtils {
|
||||||
@Test
|
@Test
|
||||||
public void testConvertUrlWithNoPort() throws URISyntaxException {
|
public void testConvertUrlWithNoPort() throws URISyntaxException {
|
||||||
Path expectedPath = new Path("hdfs://foo.com");
|
Path expectedPath = new Path("hdfs://foo.com");
|
||||||
URL url = ConverterUtils.getYarnUrlFromPath(expectedPath);
|
URL url = URL.fromPath(expectedPath);
|
||||||
Path actualPath = ConverterUtils.getPathFromYarnURL(url);
|
Path actualPath = url.toPath();
|
||||||
assertEquals(expectedPath, actualPath);
|
assertEquals(expectedPath, actualPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testConvertUrlWithUserinfo() throws URISyntaxException {
|
public void testConvertUrlWithUserinfo() throws URISyntaxException {
|
||||||
Path expectedPath = new Path("foo://username:password@example.com:8042");
|
Path expectedPath = new Path("foo://username:password@example.com:8042");
|
||||||
URL url = ConverterUtils.getYarnUrlFromPath(expectedPath);
|
URL url = URL.fromPath(expectedPath);
|
||||||
Path actualPath = ConverterUtils.getPathFromYarnURL(url);
|
Path actualPath = url.toPath();
|
||||||
assertEquals(expectedPath, actualPath);
|
assertEquals(expectedPath, actualPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testContainerId() throws URISyntaxException {
|
public void testContainerId() throws URISyntaxException {
|
||||||
ContainerId id = TestContainerId.newContainerId(0, 0, 0, 0);
|
ContainerId id = TestContainerId.newContainerId(0, 0, 0, 0);
|
||||||
String cid = ConverterUtils.toString(id);
|
String cid = id.toString();
|
||||||
assertEquals("container_0_0000_00_000000", cid);
|
assertEquals("container_0_0000_00_000000", cid);
|
||||||
ContainerId gen = ConverterUtils.toContainerId(cid);
|
ContainerId gen = ContainerId.fromString(cid);
|
||||||
assertEquals(gen, id);
|
assertEquals(gen, id);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testContainerIdWithEpoch() throws URISyntaxException {
|
public void testContainerIdWithEpoch() throws URISyntaxException {
|
||||||
ContainerId id = TestContainerId.newContainerId(0, 0, 0, 25645811);
|
ContainerId id = TestContainerId.newContainerId(0, 0, 0, 25645811);
|
||||||
String cid = ConverterUtils.toString(id);
|
String cid = id.toString();
|
||||||
assertEquals("container_0_0000_00_25645811", cid);
|
assertEquals("container_0_0000_00_25645811", cid);
|
||||||
ContainerId gen = ConverterUtils.toContainerId(cid);
|
ContainerId gen = ContainerId.fromString(cid);
|
||||||
assertEquals(gen.toString(), id.toString());
|
assertEquals(gen.toString(), id.toString());
|
||||||
|
|
||||||
long ts = System.currentTimeMillis();
|
long ts = System.currentTimeMillis();
|
||||||
ContainerId id2 =
|
ContainerId id2 =
|
||||||
TestContainerId.newContainerId(36473, 4365472, ts, 4298334883325L);
|
TestContainerId.newContainerId(36473, 4365472, ts, 4298334883325L);
|
||||||
String cid2 = ConverterUtils.toString(id2);
|
String cid2 = id2.toString();
|
||||||
assertEquals(
|
assertEquals(
|
||||||
"container_e03_" + ts + "_36473_4365472_999799999997", cid2);
|
"container_e03_" + ts + "_36473_4365472_999799999997", cid2);
|
||||||
ContainerId gen2 = ConverterUtils.toContainerId(cid2);
|
ContainerId gen2 = ContainerId.fromString(cid2);
|
||||||
assertEquals(gen2.toString(), id2.toString());
|
assertEquals(gen2.toString(), id2.toString());
|
||||||
|
|
||||||
ContainerId id3 =
|
ContainerId id3 =
|
||||||
TestContainerId.newContainerId(36473, 4365472, ts, 844424930131965L);
|
TestContainerId.newContainerId(36473, 4365472, ts, 844424930131965L);
|
||||||
String cid3 = ConverterUtils.toString(id3);
|
String cid3 = id3.toString();
|
||||||
assertEquals(
|
assertEquals(
|
||||||
"container_e767_" + ts + "_36473_4365472_1099511627773", cid3);
|
"container_e767_" + ts + "_36473_4365472_1099511627773", cid3);
|
||||||
ContainerId gen3 = ConverterUtils.toContainerId(cid3);
|
ContainerId gen3 = ContainerId.fromString(cid3);
|
||||||
assertEquals(gen3.toString(), id3.toString());
|
assertEquals(gen3.toString(), id3.toString());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@SuppressWarnings("deprecation")
|
||||||
public void testContainerIdNull() throws URISyntaxException {
|
public void testContainerIdNull() throws URISyntaxException {
|
||||||
assertNull(ConverterUtils.toString((ContainerId)null));
|
assertNull(ConverterUtils.toString((ContainerId)null));
|
||||||
}
|
}
|
||||||
|
@ -101,16 +102,19 @@ public class TestConverterUtils {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(expected = IllegalArgumentException.class)
|
@Test(expected = IllegalArgumentException.class)
|
||||||
|
@SuppressWarnings("deprecation")
|
||||||
public void testInvalidContainerId() {
|
public void testInvalidContainerId() {
|
||||||
ConverterUtils.toContainerId("container_e20_1423221031460_0003_01");
|
ContainerId.fromString("container_e20_1423221031460_0003_01");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(expected = IllegalArgumentException.class)
|
@Test(expected = IllegalArgumentException.class)
|
||||||
|
@SuppressWarnings("deprecation")
|
||||||
public void testInvalidAppattemptId() {
|
public void testInvalidAppattemptId() {
|
||||||
ConverterUtils.toApplicationAttemptId("appattempt_1423221031460");
|
ConverterUtils.toApplicationAttemptId("appattempt_1423221031460");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(expected = IllegalArgumentException.class)
|
@Test(expected = IllegalArgumentException.class)
|
||||||
|
@SuppressWarnings("deprecation")
|
||||||
public void testApplicationId() {
|
public void testApplicationId() {
|
||||||
ConverterUtils.toApplicationId("application_1423221031460");
|
ConverterUtils.toApplicationId("application_1423221031460");
|
||||||
}
|
}
|
||||||
|
|
|
@ -53,6 +53,7 @@ import java.util.zip.ZipEntry;
|
||||||
import java.util.zip.ZipOutputStream;
|
import java.util.zip.ZipOutputStream;
|
||||||
|
|
||||||
import org.apache.hadoop.util.concurrent.HadoopExecutors;
|
import org.apache.hadoop.util.concurrent.HadoopExecutors;
|
||||||
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
|
|
||||||
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
|
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
|
||||||
|
@ -103,7 +104,7 @@ public class TestFSDownload {
|
||||||
Random r, LocalResourceVisibility vis) throws IOException {
|
Random r, LocalResourceVisibility vis) throws IOException {
|
||||||
createFile(files, p, len, r);
|
createFile(files, p, len, r);
|
||||||
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
|
ret.setResource(URL.fromPath(p));
|
||||||
ret.setSize(len);
|
ret.setSize(len);
|
||||||
ret.setType(LocalResourceType.FILE);
|
ret.setType(LocalResourceType.FILE);
|
||||||
ret.setVisibility(vis);
|
ret.setVisibility(vis);
|
||||||
|
@ -134,7 +135,7 @@ public class TestFSDownload {
|
||||||
LOG.info("Done writing jar stream ");
|
LOG.info("Done writing jar stream ");
|
||||||
out.close();
|
out.close();
|
||||||
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
|
ret.setResource(URL.fromPath(p));
|
||||||
FileStatus status = files.getFileStatus(p);
|
FileStatus status = files.getFileStatus(p);
|
||||||
ret.setSize(status.getLen());
|
ret.setSize(status.getLen());
|
||||||
ret.setTimestamp(status.getModificationTime());
|
ret.setTimestamp(status.getModificationTime());
|
||||||
|
@ -162,7 +163,7 @@ public class TestFSDownload {
|
||||||
out.close();
|
out.close();
|
||||||
|
|
||||||
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
|
ret.setResource(URL.fromPath(new Path(p.toString()
|
||||||
+ ".tar")));
|
+ ".tar")));
|
||||||
ret.setSize(len);
|
ret.setSize(len);
|
||||||
ret.setType(LocalResourceType.ARCHIVE);
|
ret.setType(LocalResourceType.ARCHIVE);
|
||||||
|
@ -190,7 +191,7 @@ public class TestFSDownload {
|
||||||
out.close();
|
out.close();
|
||||||
|
|
||||||
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
|
ret.setResource(URL.fromPath(new Path(p.toString()
|
||||||
+ ".tar.gz")));
|
+ ".tar.gz")));
|
||||||
ret.setSize(len);
|
ret.setSize(len);
|
||||||
ret.setType(LocalResourceType.ARCHIVE);
|
ret.setType(LocalResourceType.ARCHIVE);
|
||||||
|
@ -216,7 +217,7 @@ public class TestFSDownload {
|
||||||
out.close();
|
out.close();
|
||||||
|
|
||||||
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
|
ret.setResource(URL.fromPath(new Path(p.toString()
|
||||||
+ ".jar")));
|
+ ".jar")));
|
||||||
ret.setSize(len);
|
ret.setSize(len);
|
||||||
ret.setType(LocalResourceType.ARCHIVE);
|
ret.setType(LocalResourceType.ARCHIVE);
|
||||||
|
@ -242,7 +243,7 @@ public class TestFSDownload {
|
||||||
out.close();
|
out.close();
|
||||||
|
|
||||||
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
|
ret.setResource(URL.fromPath(new Path(p.toString()
|
||||||
+ ".ZIP")));
|
+ ".ZIP")));
|
||||||
ret.setSize(len);
|
ret.setSize(len);
|
||||||
ret.setType(LocalResourceType.ARCHIVE);
|
ret.setType(LocalResourceType.ARCHIVE);
|
||||||
|
|
|
@ -278,7 +278,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
|
||||||
}
|
}
|
||||||
if (field == ApplicationReportField.USER_AND_ACLS) {
|
if (field == ApplicationReportField.USER_AND_ACLS) {
|
||||||
return new ApplicationReportExt(ApplicationReport.newInstance(
|
return new ApplicationReportExt(ApplicationReport.newInstance(
|
||||||
ConverterUtils.toApplicationId(entity.getEntityId()),
|
ApplicationId.fromString(entity.getEntityId()),
|
||||||
latestApplicationAttemptId, user, queue, name, null, -1, null,
|
latestApplicationAttemptId, user, queue, name, null, -1, null,
|
||||||
state, diagnosticsInfo, null, createdTime, finishedTime,
|
state, diagnosticsInfo, null, createdTime, finishedTime,
|
||||||
finalStatus, null, null, progress, type, null, appTags,
|
finalStatus, null, null, progress, type, null, appTags,
|
||||||
|
@ -394,11 +394,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
|
||||||
}
|
}
|
||||||
if (eventInfo
|
if (eventInfo
|
||||||
.containsKey(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO)) {
|
.containsKey(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO)) {
|
||||||
latestApplicationAttemptId =
|
latestApplicationAttemptId = ApplicationAttemptId.fromString(
|
||||||
ConverterUtils
|
eventInfo.get(
|
||||||
.toApplicationAttemptId(
|
|
||||||
eventInfo
|
|
||||||
.get(
|
|
||||||
ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO)
|
ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO)
|
||||||
.toString());
|
.toString());
|
||||||
}
|
}
|
||||||
|
@ -426,7 +423,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return new ApplicationReportExt(ApplicationReport.newInstance(
|
return new ApplicationReportExt(ApplicationReport.newInstance(
|
||||||
ConverterUtils.toApplicationId(entity.getEntityId()),
|
ApplicationId.fromString(entity.getEntityId()),
|
||||||
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
|
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
|
||||||
diagnosticsInfo, null, createdTime, finishedTime, finalStatus,
|
diagnosticsInfo, null, createdTime, finishedTime, finalStatus,
|
||||||
appResources, null, progress, type, null, appTags, unmanagedApplication,
|
appResources, null, progress, type, null, appTags, unmanagedApplication,
|
||||||
|
@ -471,7 +468,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
|
||||||
if (eventInfo
|
if (eventInfo
|
||||||
.containsKey(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)) {
|
.containsKey(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)) {
|
||||||
amContainerId =
|
amContainerId =
|
||||||
ConverterUtils.toContainerId(eventInfo.get(
|
ContainerId.fromString(eventInfo.get(
|
||||||
AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)
|
AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)
|
||||||
.toString());
|
.toString());
|
||||||
}
|
}
|
||||||
|
@ -513,7 +510,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
|
||||||
if (eventInfo
|
if (eventInfo
|
||||||
.containsKey(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)) {
|
.containsKey(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)) {
|
||||||
amContainerId =
|
amContainerId =
|
||||||
ConverterUtils.toContainerId(eventInfo.get(
|
ContainerId.fromString(eventInfo.get(
|
||||||
AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)
|
AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)
|
||||||
.toString());
|
.toString());
|
||||||
}
|
}
|
||||||
|
@ -521,7 +518,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return ApplicationAttemptReport.newInstance(
|
return ApplicationAttemptReport.newInstance(
|
||||||
ConverterUtils.toApplicationAttemptId(entity.getEntityId()),
|
ApplicationAttemptId.fromString(entity.getEntityId()),
|
||||||
host, rpcPort, trackingUrl, originalTrackingUrl, diagnosticsInfo,
|
host, rpcPort, trackingUrl, originalTrackingUrl, diagnosticsInfo,
|
||||||
state, amContainerId);
|
state, amContainerId);
|
||||||
}
|
}
|
||||||
|
@ -610,7 +607,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ContainerId containerId =
|
ContainerId containerId =
|
||||||
ConverterUtils.toContainerId(entity.getEntityId());
|
ContainerId.fromString(entity.getEntityId());
|
||||||
String logUrl = null;
|
String logUrl = null;
|
||||||
NodeId allocatedNode = null;
|
NodeId allocatedNode = null;
|
||||||
if (allocatedHost != null) {
|
if (allocatedHost != null) {
|
||||||
|
@ -623,7 +620,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
|
||||||
user);
|
user);
|
||||||
}
|
}
|
||||||
return ContainerReport.newInstance(
|
return ContainerReport.newInstance(
|
||||||
ConverterUtils.toContainerId(entity.getEntityId()),
|
ContainerId.fromString(entity.getEntityId()),
|
||||||
Resource.newInstance(allocatedMem, allocatedVcore), allocatedNode,
|
Resource.newInstance(allocatedMem, allocatedVcore), allocatedNode,
|
||||||
Priority.newInstance(allocatedPriority),
|
Priority.newInstance(allocatedPriority),
|
||||||
createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state,
|
createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state,
|
||||||
|
|
|
@ -204,7 +204,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService
|
||||||
FileStatus[] files = fs.listStatus(rootDirPath);
|
FileStatus[] files = fs.listStatus(rootDirPath);
|
||||||
for (FileStatus file : files) {
|
for (FileStatus file : files) {
|
||||||
ApplicationId appId =
|
ApplicationId appId =
|
||||||
ConverterUtils.toApplicationId(file.getPath().getName());
|
ApplicationId.fromString(file.getPath().getName());
|
||||||
try {
|
try {
|
||||||
ApplicationHistoryData historyData = getApplication(appId);
|
ApplicationHistoryData historyData = getApplication(appId);
|
||||||
if (historyData != null) {
|
if (historyData != null) {
|
||||||
|
@ -231,8 +231,8 @@ public class FileSystemApplicationHistoryStore extends AbstractService
|
||||||
HistoryFileReader.Entry entry = hfReader.next();
|
HistoryFileReader.Entry entry = hfReader.next();
|
||||||
if (entry.key.id.startsWith(
|
if (entry.key.id.startsWith(
|
||||||
ConverterUtils.APPLICATION_ATTEMPT_PREFIX)) {
|
ConverterUtils.APPLICATION_ATTEMPT_PREFIX)) {
|
||||||
ApplicationAttemptId appAttemptId =
|
ApplicationAttemptId appAttemptId = ApplicationAttemptId.fromString(
|
||||||
ConverterUtils.toApplicationAttemptId(entry.key.id);
|
entry.key.id);
|
||||||
if (appAttemptId.getApplicationId().equals(appId)) {
|
if (appAttemptId.getApplicationId().equals(appId)) {
|
||||||
ApplicationAttemptHistoryData historyData =
|
ApplicationAttemptHistoryData historyData =
|
||||||
historyDataMap.get(appAttemptId);
|
historyDataMap.get(appAttemptId);
|
||||||
|
@ -385,7 +385,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService
|
||||||
HistoryFileReader.Entry entry = hfReader.next();
|
HistoryFileReader.Entry entry = hfReader.next();
|
||||||
if (entry.key.id.startsWith(ConverterUtils.CONTAINER_PREFIX)) {
|
if (entry.key.id.startsWith(ConverterUtils.CONTAINER_PREFIX)) {
|
||||||
ContainerId containerId =
|
ContainerId containerId =
|
||||||
ConverterUtils.toContainerId(entry.key.id);
|
ContainerId.fromString(entry.key.id);
|
||||||
if (containerId.getApplicationAttemptId().equals(appAttemptId)) {
|
if (containerId.getApplicationAttemptId().equals(appAttemptId)) {
|
||||||
ContainerHistoryData historyData =
|
ContainerHistoryData historyData =
|
||||||
historyDataMap.get(containerId);
|
historyDataMap.get(containerId);
|
||||||
|
|
|
@ -113,7 +113,7 @@ public class BuilderUtils {
|
||||||
public static LocalResource newLocalResource(URI uri,
|
public static LocalResource newLocalResource(URI uri,
|
||||||
LocalResourceType type, LocalResourceVisibility visibility, long size,
|
LocalResourceType type, LocalResourceVisibility visibility, long size,
|
||||||
long timestamp, boolean shouldBeUploadedToSharedCache) {
|
long timestamp, boolean shouldBeUploadedToSharedCache) {
|
||||||
return newLocalResource(ConverterUtils.getYarnUrlFromURI(uri), type,
|
return newLocalResource(URL.fromURI(uri), type,
|
||||||
visibility, size, timestamp, shouldBeUploadedToSharedCache);
|
visibility, size, timestamp, shouldBeUploadedToSharedCache);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -65,7 +65,7 @@ public class AppAttemptBlock extends HtmlBlock {
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
appAttemptId = ConverterUtils.toApplicationAttemptId(attemptid);
|
appAttemptId = ApplicationAttemptId.fromString(attemptid);
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
puts("Invalid application attempt ID: " + attemptid);
|
puts("Invalid application attempt ID: " + attemptid);
|
||||||
return;
|
return;
|
||||||
|
|
|
@ -59,7 +59,7 @@ public class ContainerBlock extends HtmlBlock {
|
||||||
|
|
||||||
ContainerId containerId = null;
|
ContainerId containerId = null;
|
||||||
try {
|
try {
|
||||||
containerId = ConverterUtils.toContainerId(containerid);
|
containerId = ContainerId.fromString(containerid);
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
puts("Invalid container ID: " + containerid);
|
puts("Invalid container ID: " + containerid);
|
||||||
return;
|
return;
|
||||||
|
|
|
@ -431,7 +431,7 @@ public class WebServices {
|
||||||
}
|
}
|
||||||
ApplicationId aid = null;
|
ApplicationId aid = null;
|
||||||
try {
|
try {
|
||||||
aid = ConverterUtils.toApplicationId(appId);
|
aid = ApplicationId.fromString(appId);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new BadRequestException(e);
|
throw new BadRequestException(e);
|
||||||
}
|
}
|
||||||
|
@ -449,7 +449,7 @@ public class WebServices {
|
||||||
}
|
}
|
||||||
ApplicationAttemptId aaid = null;
|
ApplicationAttemptId aaid = null;
|
||||||
try {
|
try {
|
||||||
aaid = ConverterUtils.toApplicationAttemptId(appAttemptId);
|
aaid = ApplicationAttemptId.fromString(appAttemptId);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new BadRequestException(e);
|
throw new BadRequestException(e);
|
||||||
}
|
}
|
||||||
|
@ -466,7 +466,7 @@ public class WebServices {
|
||||||
}
|
}
|
||||||
ContainerId cid = null;
|
ContainerId cid = null;
|
||||||
try {
|
try {
|
||||||
cid = ConverterUtils.toContainerId(containerId);
|
cid = ContainerId.fromString(containerId);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new BadRequestException(e);
|
throw new BadRequestException(e);
|
||||||
}
|
}
|
||||||
|
|
|
@ -167,11 +167,10 @@ public class DefaultContainerExecutor extends ContainerExecutor {
|
||||||
ContainerId containerId = container.getContainerId();
|
ContainerId containerId = container.getContainerId();
|
||||||
|
|
||||||
// create container dirs on all disks
|
// create container dirs on all disks
|
||||||
String containerIdStr = ConverterUtils.toString(containerId);
|
String containerIdStr = containerId.toString();
|
||||||
String appIdStr =
|
String appIdStr =
|
||||||
ConverterUtils.toString(
|
|
||||||
containerId.getApplicationAttemptId().
|
containerId.getApplicationAttemptId().
|
||||||
getApplicationId());
|
getApplicationId().toString();
|
||||||
for (String sLocalDir : localDirs) {
|
for (String sLocalDir : localDirs) {
|
||||||
Path usersdir = new Path(sLocalDir, ContainerLocalizer.USERCACHE);
|
Path usersdir = new Path(sLocalDir, ContainerLocalizer.USERCACHE);
|
||||||
Path userdir = new Path(usersdir, user);
|
Path userdir = new Path(usersdir, user);
|
||||||
|
|
|
@ -195,9 +195,9 @@ public class DockerContainerExecutor extends ContainerExecutor {
|
||||||
ContainerId containerId = container.getContainerId();
|
ContainerId containerId = container.getContainerId();
|
||||||
|
|
||||||
// create container dirs on all disks
|
// create container dirs on all disks
|
||||||
String containerIdStr = ConverterUtils.toString(containerId);
|
String containerIdStr = containerId.toString();
|
||||||
String appIdStr = ConverterUtils.toString(
|
String appIdStr =
|
||||||
containerId.getApplicationAttemptId().getApplicationId());
|
containerId.getApplicationAttemptId().getApplicationId().toString();
|
||||||
for (String sLocalDir : localDirs) {
|
for (String sLocalDir : localDirs) {
|
||||||
Path usersdir = new Path(sLocalDir, ContainerLocalizer.USERCACHE);
|
Path usersdir = new Path(sLocalDir, ContainerLocalizer.USERCACHE);
|
||||||
Path userdir = new Path(usersdir, userName);
|
Path userdir = new Path(usersdir, userName);
|
||||||
|
|
|
@ -319,7 +319,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
|
||||||
String runAsUser = getRunAsUser(user);
|
String runAsUser = getRunAsUser(user);
|
||||||
|
|
||||||
ContainerId containerId = container.getContainerId();
|
ContainerId containerId = container.getContainerId();
|
||||||
String containerIdStr = ConverterUtils.toString(containerId);
|
String containerIdStr = containerId.toString();
|
||||||
|
|
||||||
resourcesHandler.preExecute(containerId,
|
resourcesHandler.preExecute(containerId,
|
||||||
container.getResource());
|
container.getResource());
|
||||||
|
|
|
@ -1361,7 +1361,7 @@ public class ContainerImpl implements Container {
|
||||||
public String toString() {
|
public String toString() {
|
||||||
this.readLock.lock();
|
this.readLock.lock();
|
||||||
try {
|
try {
|
||||||
return ConverterUtils.toString(this.containerId);
|
return this.containerId.toString();
|
||||||
} finally {
|
} finally {
|
||||||
this.readLock.unlock();
|
this.readLock.unlock();
|
||||||
}
|
}
|
||||||
|
|
|
@ -163,7 +163,7 @@ public class ContainerLaunch implements Callable<Integer> {
|
||||||
|
|
||||||
final ContainerLaunchContext launchContext = container.getLaunchContext();
|
final ContainerLaunchContext launchContext = container.getLaunchContext();
|
||||||
ContainerId containerID = container.getContainerId();
|
ContainerId containerID = container.getContainerId();
|
||||||
String containerIdStr = ConverterUtils.toString(containerID);
|
String containerIdStr = containerID.toString();
|
||||||
final List<String> command = launchContext.getCommands();
|
final List<String> command = launchContext.getCommands();
|
||||||
int ret = -1;
|
int ret = -1;
|
||||||
|
|
||||||
|
@ -326,7 +326,7 @@ public class ContainerLaunch implements Callable<Integer> {
|
||||||
protected List<String> getContainerLogDirs(List<String> logDirs) {
|
protected List<String> getContainerLogDirs(List<String> logDirs) {
|
||||||
List<String> containerLogDirs = new ArrayList<>(logDirs.size());
|
List<String> containerLogDirs = new ArrayList<>(logDirs.size());
|
||||||
String appIdStr = app.getAppId().toString();
|
String appIdStr = app.getAppId().toString();
|
||||||
String containerIdStr = ConverterUtils.toString(container.getContainerId());
|
String containerIdStr = container.getContainerId().toString();
|
||||||
String relativeContainerLogDir = ContainerLaunch
|
String relativeContainerLogDir = ContainerLaunch
|
||||||
.getRelativeContainerLogDir(appIdStr, containerIdStr);
|
.getRelativeContainerLogDir(appIdStr, containerIdStr);
|
||||||
|
|
||||||
|
@ -520,7 +520,7 @@ public class ContainerLaunch implements Callable<Integer> {
|
||||||
@SuppressWarnings("unchecked") // dispatcher not typed
|
@SuppressWarnings("unchecked") // dispatcher not typed
|
||||||
public void cleanupContainer() throws IOException {
|
public void cleanupContainer() throws IOException {
|
||||||
ContainerId containerId = container.getContainerId();
|
ContainerId containerId = container.getContainerId();
|
||||||
String containerIdStr = ConverterUtils.toString(containerId);
|
String containerIdStr = containerId.toString();
|
||||||
LOG.info("Cleaning up container " + containerIdStr);
|
LOG.info("Cleaning up container " + containerIdStr);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
@ -616,7 +616,7 @@ public class ContainerLaunch implements Callable<Integer> {
|
||||||
throws IOException {
|
throws IOException {
|
||||||
ContainerId containerId =
|
ContainerId containerId =
|
||||||
container.getContainerTokenIdentifier().getContainerID();
|
container.getContainerTokenIdentifier().getContainerID();
|
||||||
String containerIdStr = ConverterUtils.toString(containerId);
|
String containerIdStr = containerId.toString();
|
||||||
String user = container.getUser();
|
String user = container.getUser();
|
||||||
Signal signal = translateCommandToSignal(command);
|
Signal signal = translateCommandToSignal(command);
|
||||||
if (signal.equals(Signal.NULL)) {
|
if (signal.equals(Signal.NULL)) {
|
||||||
|
@ -708,7 +708,7 @@ public class ContainerLaunch implements Callable<Integer> {
|
||||||
*/
|
*/
|
||||||
private String getContainerPid(Path pidFilePath) throws Exception {
|
private String getContainerPid(Path pidFilePath) throws Exception {
|
||||||
String containerIdStr =
|
String containerIdStr =
|
||||||
ConverterUtils.toString(container.getContainerId());
|
container.getContainerId().toString();
|
||||||
String processId = null;
|
String processId = null;
|
||||||
LOG.debug("Accessing pid for container " + containerIdStr
|
LOG.debug("Accessing pid for container " + containerIdStr
|
||||||
+ " from pid file " + pidFilePath);
|
+ " from pid file " + pidFilePath);
|
||||||
|
|
|
@ -65,7 +65,7 @@ public class ContainerRelaunch extends ContainerLaunch {
|
||||||
}
|
}
|
||||||
|
|
||||||
ContainerId containerId = container.getContainerId();
|
ContainerId containerId = container.getContainerId();
|
||||||
String containerIdStr = ConverterUtils.toString(containerId);
|
String containerIdStr = containerId.toString();
|
||||||
int ret = -1;
|
int ret = -1;
|
||||||
Path containerLogDir;
|
Path containerLogDir;
|
||||||
try {
|
try {
|
||||||
|
|
|
@ -68,9 +68,9 @@ public class RecoveredContainerLaunch extends ContainerLaunch {
|
||||||
public Integer call() {
|
public Integer call() {
|
||||||
int retCode = ExitCode.LOST.getExitCode();
|
int retCode = ExitCode.LOST.getExitCode();
|
||||||
ContainerId containerId = container.getContainerId();
|
ContainerId containerId = container.getContainerId();
|
||||||
String appIdStr = ConverterUtils.toString(
|
String appIdStr =
|
||||||
containerId.getApplicationAttemptId().getApplicationId());
|
containerId.getApplicationAttemptId().getApplicationId().toString();
|
||||||
String containerIdStr = ConverterUtils.toString(containerId);
|
String containerIdStr = containerId.toString();
|
||||||
|
|
||||||
dispatcher.getEventHandler().handle(new ContainerEvent(containerId,
|
dispatcher.getEventHandler().handle(new ContainerEvent(containerId,
|
||||||
ContainerEventType.CONTAINER_LAUNCHED));
|
ContainerEventType.CONTAINER_LAUNCHED));
|
||||||
|
|
|
@ -56,6 +56,7 @@ import org.apache.hadoop.util.concurrent.HadoopExecutors;
|
||||||
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
|
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResource;
|
import org.apache.hadoop.yarn.api.records.LocalResource;
|
||||||
import org.apache.hadoop.yarn.api.records.SerializedException;
|
import org.apache.hadoop.yarn.api.records.SerializedException;
|
||||||
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
|
@ -295,7 +296,7 @@ public class ContainerLocalizer {
|
||||||
try {
|
try {
|
||||||
Path localPath = fPath.get();
|
Path localPath = fPath.get();
|
||||||
stat.setLocalPath(
|
stat.setLocalPath(
|
||||||
ConverterUtils.getYarnUrlFromPath(localPath));
|
URL.fromPath(localPath));
|
||||||
stat.setLocalSize(
|
stat.setLocalSize(
|
||||||
FileUtil.getDU(new File(localPath.getParent().toUri())));
|
FileUtil.getDU(new File(localPath.getParent().toUri())));
|
||||||
stat.setStatus(ResourceStatusType.FETCH_SUCCESS);
|
stat.setStatus(ResourceStatusType.FETCH_SUCCESS);
|
||||||
|
|
|
@ -43,7 +43,7 @@ public class LocalResourceRequest
|
||||||
*/
|
*/
|
||||||
public LocalResourceRequest(LocalResource resource)
|
public LocalResourceRequest(LocalResource resource)
|
||||||
throws URISyntaxException {
|
throws URISyntaxException {
|
||||||
this(ConverterUtils.getPathFromYarnURL(resource.getResource()),
|
this(resource.getResource().toPath(),
|
||||||
resource.getTimestamp(),
|
resource.getTimestamp(),
|
||||||
resource.getType(),
|
resource.getType(),
|
||||||
resource.getVisibility(),
|
resource.getVisibility(),
|
||||||
|
@ -133,7 +133,7 @@ public class LocalResourceRequest
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public URL getResource() {
|
public URL getResource() {
|
||||||
return ConverterUtils.getYarnUrlFromPath(loc);
|
return URL.fromPath(loc);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -79,6 +79,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResource;
|
import org.apache.hadoop.yarn.api.records.LocalResource;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
||||||
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.event.Dispatcher;
|
import org.apache.hadoop.yarn.event.Dispatcher;
|
||||||
|
@ -301,7 +302,7 @@ public class ResourceLocalizationService extends CompositeService
|
||||||
trackerState = appEntry.getValue();
|
trackerState = appEntry.getValue();
|
||||||
if (!trackerState.isEmpty()) {
|
if (!trackerState.isEmpty()) {
|
||||||
ApplicationId appId = appEntry.getKey();
|
ApplicationId appId = appEntry.getKey();
|
||||||
String appIdStr = ConverterUtils.toString(appId);
|
String appIdStr = appId.toString();
|
||||||
LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
|
LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
|
||||||
appId, dispatcher, false, super.getConfig(), stateStore);
|
appId, dispatcher, false, super.getConfig(), stateStore);
|
||||||
LocalResourcesTracker oldTracker = appRsrc.putIfAbsent(appIdStr,
|
LocalResourcesTracker oldTracker = appRsrc.putIfAbsent(appIdStr,
|
||||||
|
@ -442,7 +443,7 @@ public class ResourceLocalizationService extends CompositeService
|
||||||
String userName = app.getUser();
|
String userName = app.getUser();
|
||||||
privateRsrc.putIfAbsent(userName, new LocalResourcesTrackerImpl(userName,
|
privateRsrc.putIfAbsent(userName, new LocalResourcesTrackerImpl(userName,
|
||||||
null, dispatcher, true, super.getConfig(), stateStore));
|
null, dispatcher, true, super.getConfig(), stateStore));
|
||||||
String appIdStr = ConverterUtils.toString(app.getAppId());
|
String appIdStr = app.getAppId().toString();
|
||||||
appRsrc.putIfAbsent(appIdStr, new LocalResourcesTrackerImpl(app.getUser(),
|
appRsrc.putIfAbsent(appIdStr, new LocalResourcesTrackerImpl(app.getUser(),
|
||||||
app.getAppId(), dispatcher, false, super.getConfig(), stateStore));
|
app.getAppId(), dispatcher, false, super.getConfig(), stateStore));
|
||||||
// 1) Signal container init
|
// 1) Signal container init
|
||||||
|
@ -491,7 +492,7 @@ public class ResourceLocalizationService extends CompositeService
|
||||||
private void handleContainerResourcesLocalized(
|
private void handleContainerResourcesLocalized(
|
||||||
ContainerLocalizationEvent event) {
|
ContainerLocalizationEvent event) {
|
||||||
Container c = event.getContainer();
|
Container c = event.getContainer();
|
||||||
String locId = ConverterUtils.toString(c.getContainerId());
|
String locId = c.getContainerId().toString();
|
||||||
localizerTracker.endContainerLocalization(locId);
|
localizerTracker.endContainerLocalization(locId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -528,14 +529,15 @@ public class ResourceLocalizationService extends CompositeService
|
||||||
c.getContainerId()));
|
c.getContainerId()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
String locId = ConverterUtils.toString(c.getContainerId());
|
String locId = c.getContainerId().toString();
|
||||||
localizerTracker.cleanupPrivLocalizers(locId);
|
localizerTracker.cleanupPrivLocalizers(locId);
|
||||||
|
|
||||||
// Delete the container directories
|
// Delete the container directories
|
||||||
String userName = c.getUser();
|
String userName = c.getUser();
|
||||||
String containerIDStr = c.toString();
|
String containerIDStr = c.toString();
|
||||||
String appIDStr = ConverterUtils.toString(
|
String appIDStr =
|
||||||
c.getContainerId().getApplicationAttemptId().getApplicationId());
|
c.getContainerId().getApplicationAttemptId().getApplicationId()
|
||||||
|
.toString();
|
||||||
|
|
||||||
// Try deleting from good local dirs and full local dirs because a dir might
|
// Try deleting from good local dirs and full local dirs because a dir might
|
||||||
// have gone bad while the app was running(disk full). In addition
|
// have gone bad while the app was running(disk full). In addition
|
||||||
|
@ -583,7 +585,7 @@ public class ResourceLocalizationService extends CompositeService
|
||||||
ApplicationId appId = application.getAppId();
|
ApplicationId appId = application.getAppId();
|
||||||
String appIDStr = application.toString();
|
String appIDStr = application.toString();
|
||||||
LocalResourcesTracker appLocalRsrcsTracker =
|
LocalResourcesTracker appLocalRsrcsTracker =
|
||||||
appRsrc.remove(ConverterUtils.toString(appId));
|
appRsrc.remove(appId.toString());
|
||||||
if (appLocalRsrcsTracker != null) {
|
if (appLocalRsrcsTracker != null) {
|
||||||
for (LocalizedResource rsrc : appLocalRsrcsTracker ) {
|
for (LocalizedResource rsrc : appLocalRsrcsTracker ) {
|
||||||
Path localPath = rsrc.getLocalPath();
|
Path localPath = rsrc.getLocalPath();
|
||||||
|
@ -637,7 +639,7 @@ public class ResourceLocalizationService extends CompositeService
|
||||||
case PRIVATE:
|
case PRIVATE:
|
||||||
return privateRsrc.get(user);
|
return privateRsrc.get(user);
|
||||||
case APPLICATION:
|
case APPLICATION:
|
||||||
return appRsrc.get(ConverterUtils.toString(appId));
|
return appRsrc.get(appId.toString());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -977,7 +979,7 @@ public class ResourceLocalizationService extends CompositeService
|
||||||
LocalResourceRequest nextRsrc = nRsrc.getRequest();
|
LocalResourceRequest nextRsrc = nRsrc.getRequest();
|
||||||
LocalResource next =
|
LocalResource next =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
next.setResource(ConverterUtils.getYarnUrlFromPath(nextRsrc
|
next.setResource(URL.fromPath(nextRsrc
|
||||||
.getPath()));
|
.getPath()));
|
||||||
next.setTimestamp(nextRsrc.getTimestamp());
|
next.setTimestamp(nextRsrc.getTimestamp());
|
||||||
next.setType(nextRsrc.getType());
|
next.setType(nextRsrc.getType());
|
||||||
|
@ -1028,8 +1030,8 @@ public class ResourceLocalizationService extends CompositeService
|
||||||
try {
|
try {
|
||||||
getLocalResourcesTracker(req.getVisibility(), user, applicationId)
|
getLocalResourcesTracker(req.getVisibility(), user, applicationId)
|
||||||
.handle(
|
.handle(
|
||||||
new ResourceLocalizedEvent(req, ConverterUtils
|
new ResourceLocalizedEvent(req, stat.getLocalPath().toPath(),
|
||||||
.getPathFromYarnURL(stat.getLocalPath()), stat.getLocalSize()));
|
stat.getLocalSize()));
|
||||||
} catch (URISyntaxException e) { }
|
} catch (URISyntaxException e) { }
|
||||||
|
|
||||||
// unlocking the resource and removing it from scheduled resource
|
// unlocking the resource and removing it from scheduled resource
|
||||||
|
@ -1142,8 +1144,8 @@ public class ResourceLocalizationService extends CompositeService
|
||||||
.setNmPrivateContainerTokens(nmPrivateCTokensPath)
|
.setNmPrivateContainerTokens(nmPrivateCTokensPath)
|
||||||
.setNmAddr(localizationServerAddress)
|
.setNmAddr(localizationServerAddress)
|
||||||
.setUser(context.getUser())
|
.setUser(context.getUser())
|
||||||
.setAppId(ConverterUtils.toString(context.getContainerId()
|
.setAppId(context.getContainerId()
|
||||||
.getApplicationAttemptId().getApplicationId()))
|
.getApplicationAttemptId().getApplicationId().toString())
|
||||||
.setLocId(localizerId)
|
.setLocId(localizerId)
|
||||||
.setDirsHandler(dirsHandler)
|
.setDirsHandler(dirsHandler)
|
||||||
.build());
|
.build());
|
||||||
|
|
|
@ -37,7 +37,7 @@ public class LocalizerResourceRequestEvent extends LocalizerEvent {
|
||||||
public LocalizerResourceRequestEvent(LocalizedResource resource,
|
public LocalizerResourceRequestEvent(LocalizedResource resource,
|
||||||
LocalResourceVisibility vis, LocalizerContext context, String pattern) {
|
LocalResourceVisibility vis, LocalizerContext context, String pattern) {
|
||||||
super(LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION,
|
super(LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION,
|
||||||
ConverterUtils.toString(context.getContainerId()));
|
context.getContainerId().toString());
|
||||||
this.vis = vis;
|
this.vis = vis;
|
||||||
this.context = context;
|
this.context = context;
|
||||||
this.resource = resource;
|
this.resource = resource;
|
||||||
|
|
|
@ -211,7 +211,7 @@ class SharedCacheUploader implements Callable<Boolean> {
|
||||||
|
|
||||||
final Path remotePath;
|
final Path remotePath;
|
||||||
try {
|
try {
|
||||||
remotePath = ConverterUtils.getPathFromYarnURL(resource.getResource());
|
remotePath = resource.getResource().toPath();
|
||||||
} catch (URISyntaxException e) {
|
} catch (URISyntaxException e) {
|
||||||
throw new IOException("Invalid resource", e);
|
throw new IOException("Invalid resource", e);
|
||||||
}
|
}
|
||||||
|
|
|
@ -164,7 +164,7 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
|
||||||
this.conf = conf;
|
this.conf = conf;
|
||||||
this.delService = deletionService;
|
this.delService = deletionService;
|
||||||
this.appId = appId;
|
this.appId = appId;
|
||||||
this.applicationId = ConverterUtils.toString(appId);
|
this.applicationId = appId.toString();
|
||||||
this.userUgi = userUgi;
|
this.userUgi = userUgi;
|
||||||
this.dirsHandler = dirsHandler;
|
this.dirsHandler = dirsHandler;
|
||||||
this.remoteNodeLogFileForApp = remoteNodeLogFileForApp;
|
this.remoteNodeLogFileForApp = remoteNodeLogFileForApp;
|
||||||
|
|
|
@ -186,7 +186,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
if (idEndPos < 0) {
|
if (idEndPos < 0) {
|
||||||
throw new IOException("Unable to determine container in key: " + key);
|
throw new IOException("Unable to determine container in key: " + key);
|
||||||
}
|
}
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(
|
ContainerId containerId = ContainerId.fromString(
|
||||||
key.substring(CONTAINERS_KEY_PREFIX.length(), idEndPos));
|
key.substring(CONTAINERS_KEY_PREFIX.length(), idEndPos));
|
||||||
String keyPrefix = key.substring(0, idEndPos+1);
|
String keyPrefix = key.substring(0, idEndPos+1);
|
||||||
RecoveredContainerState rcs = loadContainerState(containerId,
|
RecoveredContainerState rcs = loadContainerState(containerId,
|
||||||
|
@ -654,7 +654,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
throw new IOException("Unable to determine appID in resource key: "
|
throw new IOException("Unable to determine appID in resource key: "
|
||||||
+ key);
|
+ key);
|
||||||
}
|
}
|
||||||
ApplicationId appId = ConverterUtils.toApplicationId(
|
ApplicationId appId = ApplicationId.fromString(
|
||||||
key.substring(appIdStartPos, appIdEndPos));
|
key.substring(appIdStartPos, appIdEndPos));
|
||||||
userResources.appTrackerStates.put(appId,
|
userResources.appTrackerStates.put(appId,
|
||||||
loadResourceTrackerState(iter, key.substring(0, appIdEndPos+1)));
|
loadResourceTrackerState(iter, key.substring(0, appIdEndPos+1)));
|
||||||
|
@ -822,7 +822,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
ApplicationAttemptId.appAttemptIdStrPrefix)) {
|
ApplicationAttemptId.appAttemptIdStrPrefix)) {
|
||||||
ApplicationAttemptId attempt;
|
ApplicationAttemptId attempt;
|
||||||
try {
|
try {
|
||||||
attempt = ConverterUtils.toApplicationAttemptId(key);
|
attempt = ApplicationAttemptId.fromString(key);
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
throw new IOException("Bad application master key state for "
|
throw new IOException("Bad application master key state for "
|
||||||
+ fullKey, e);
|
+ fullKey, e);
|
||||||
|
@ -926,7 +926,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
ContainerId containerId;
|
ContainerId containerId;
|
||||||
Long expTime;
|
Long expTime;
|
||||||
try {
|
try {
|
||||||
containerId = ConverterUtils.toContainerId(containerIdStr);
|
containerId = ContainerId.fromString(containerIdStr);
|
||||||
expTime = Long.parseLong(asString(value));
|
expTime = Long.parseLong(asString(value));
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
throw new IOException("Bad container token state for " + key, e);
|
throw new IOException("Bad container token state for " + key, e);
|
||||||
|
@ -988,7 +988,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
|
||||||
String appIdStr = fullKey.substring(logDeleterKeyPrefixLength);
|
String appIdStr = fullKey.substring(logDeleterKeyPrefixLength);
|
||||||
ApplicationId appId = null;
|
ApplicationId appId = null;
|
||||||
try {
|
try {
|
||||||
appId = ConverterUtils.toApplicationId(appIdStr);
|
appId = ApplicationId.fromString(appIdStr);
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
LOG.warn("Skipping unknown log deleter key " + fullKey);
|
LOG.warn("Skipping unknown log deleter key " + fullKey);
|
||||||
continue;
|
continue;
|
||||||
|
|
|
@ -28,7 +28,7 @@ public class NodeManagerBuilderUtils {
|
||||||
|
|
||||||
public static ResourceLocalizationSpec newResourceLocalizationSpec(
|
public static ResourceLocalizationSpec newResourceLocalizationSpec(
|
||||||
LocalResource rsrc, Path path) {
|
LocalResource rsrc, Path path) {
|
||||||
URL local = ConverterUtils.getYarnUrlFromPath(path);
|
URL local = URL.fromPath(path);
|
||||||
ResourceLocalizationSpec resourceLocalizationSpec =
|
ResourceLocalizationSpec resourceLocalizationSpec =
|
||||||
Records.newRecord(ResourceLocalizationSpec.class);
|
Records.newRecord(ResourceLocalizationSpec.class);
|
||||||
resourceLocalizationSpec.setDestinationDirectory(local);
|
resourceLocalizationSpec.setDestinationDirectory(local);
|
||||||
|
|
|
@ -27,6 +27,7 @@ import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.util.Shell;
|
import org.apache.hadoop.util.Shell;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -69,7 +70,7 @@ public class ProcessIdFileReader {
|
||||||
// On Windows, pid is expected to be a container ID, so find first
|
// On Windows, pid is expected to be a container ID, so find first
|
||||||
// line that parses successfully as a container ID.
|
// line that parses successfully as a container ID.
|
||||||
try {
|
try {
|
||||||
ConverterUtils.toContainerId(temp);
|
ContainerId.fromString(temp);
|
||||||
processId = temp;
|
processId = temp;
|
||||||
break;
|
break;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
|
|
@ -76,10 +76,9 @@ public class ApplicationPage extends NMView implements YarnWebParams {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void render(Block html) {
|
protected void render(Block html) {
|
||||||
ApplicationId applicationID = null;
|
ApplicationId applicationID;
|
||||||
try {
|
try {
|
||||||
applicationID = ConverterUtils.toApplicationId(this.recordFactory,
|
applicationID = ApplicationId.fromString($(APPLICATION_ID));
|
||||||
$(APPLICATION_ID));
|
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
html.p()._("Invalid Application Id " + $(APPLICATION_ID))._();
|
html.p()._("Invalid Application Id " + $(APPLICATION_ID))._();
|
||||||
return;
|
return;
|
||||||
|
|
|
@ -92,7 +92,7 @@ public class ContainerLogsPage extends NMView {
|
||||||
|
|
||||||
ContainerId containerId;
|
ContainerId containerId;
|
||||||
try {
|
try {
|
||||||
containerId = ConverterUtils.toContainerId($(CONTAINER_ID));
|
containerId = ContainerId.fromString($(CONTAINER_ID));
|
||||||
} catch (IllegalArgumentException ex) {
|
} catch (IllegalArgumentException ex) {
|
||||||
html.h1("Invalid container ID: " + $(CONTAINER_ID));
|
html.h1("Invalid container ID: " + $(CONTAINER_ID));
|
||||||
return;
|
return;
|
||||||
|
|
|
@ -78,8 +78,8 @@ public class ContainerLogsUtils {
|
||||||
List<File> containerLogDirs = new ArrayList<File>(logDirs.size());
|
List<File> containerLogDirs = new ArrayList<File>(logDirs.size());
|
||||||
for (String logDir : logDirs) {
|
for (String logDir : logDirs) {
|
||||||
logDir = new File(logDir).toURI().getPath();
|
logDir = new File(logDir).toURI().getPath();
|
||||||
String appIdStr = ConverterUtils.toString(containerId
|
String appIdStr = containerId
|
||||||
.getApplicationAttemptId().getApplicationId());
|
.getApplicationAttemptId().getApplicationId().toString();
|
||||||
File appLogDir = new File(logDir, appIdStr);
|
File appLogDir = new File(logDir, appIdStr);
|
||||||
containerLogDirs.add(new File(appLogDir, containerId.toString()));
|
containerLogDirs.add(new File(appLogDir, containerId.toString()));
|
||||||
}
|
}
|
||||||
|
@ -160,7 +160,7 @@ public class ContainerLogsUtils {
|
||||||
|
|
||||||
public static FileInputStream openLogFileForRead(String containerIdStr, File logFile,
|
public static FileInputStream openLogFileForRead(String containerIdStr, File logFile,
|
||||||
Context context) throws IOException {
|
Context context) throws IOException {
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
ApplicationId applicationId = containerId.getApplicationAttemptId()
|
ApplicationId applicationId = containerId.getApplicationAttemptId()
|
||||||
.getApplicationId();
|
.getApplicationId();
|
||||||
String user = context.getApplications().get(
|
String user = context.getApplications().get(
|
||||||
|
|
|
@ -63,7 +63,7 @@ public class ContainerPage extends NMView implements YarnWebParams {
|
||||||
protected void render(Block html) {
|
protected void render(Block html) {
|
||||||
ContainerId containerID;
|
ContainerId containerID;
|
||||||
try {
|
try {
|
||||||
containerID = ConverterUtils.toContainerId($(CONTAINER_ID));
|
containerID = ContainerId.fromString($(CONTAINER_ID));
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
html.p()._("Invalid containerId " + $(CONTAINER_ID))._();
|
html.p()._("Invalid containerId " + $(CONTAINER_ID))._();
|
||||||
return;
|
return;
|
||||||
|
|
|
@ -181,7 +181,7 @@ public class NMWebServices {
|
||||||
ContainerId containerId = null;
|
ContainerId containerId = null;
|
||||||
init();
|
init();
|
||||||
try {
|
try {
|
||||||
containerId = ConverterUtils.toContainerId(id);
|
containerId = ContainerId.fromString(id);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new BadRequestException("invalid container id, " + id);
|
throw new BadRequestException("invalid container id, " + id);
|
||||||
}
|
}
|
||||||
|
@ -224,7 +224,7 @@ public class NMWebServices {
|
||||||
@QueryParam("size") String size) {
|
@QueryParam("size") String size) {
|
||||||
ContainerId containerId;
|
ContainerId containerId;
|
||||||
try {
|
try {
|
||||||
containerId = ConverterUtils.toContainerId(containerIdStr);
|
containerId = ContainerId.fromString(containerIdStr);
|
||||||
} catch (IllegalArgumentException ex) {
|
} catch (IllegalArgumentException ex) {
|
||||||
return Response.status(Status.BAD_REQUEST).build();
|
return Response.status(Status.BAD_REQUEST).build();
|
||||||
}
|
}
|
||||||
|
|
|
@ -42,14 +42,14 @@ public class AppInfo {
|
||||||
} // JAXB needs this
|
} // JAXB needs this
|
||||||
|
|
||||||
public AppInfo(final Application app) {
|
public AppInfo(final Application app) {
|
||||||
this.id = ConverterUtils.toString(app.getAppId());
|
this.id = app.getAppId().toString();
|
||||||
this.state = app.getApplicationState().toString();
|
this.state = app.getApplicationState().toString();
|
||||||
this.user = app.getUser();
|
this.user = app.getUser();
|
||||||
|
|
||||||
this.containerids = new ArrayList<String>();
|
this.containerids = new ArrayList<String>();
|
||||||
Map<ContainerId, Container> appContainers = app.getContainers();
|
Map<ContainerId, Container> appContainers = app.getContainers();
|
||||||
for (ContainerId containerId : appContainers.keySet()) {
|
for (ContainerId containerId : appContainers.keySet()) {
|
||||||
String containerIdStr = ConverterUtils.toString(containerId);
|
String containerIdStr = containerId.toString();
|
||||||
containerids.add(containerIdStr);
|
containerids.add(containerIdStr);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -118,7 +118,7 @@ public class TestNodeManagerReboot {
|
||||||
ContainerId cId = createContainerId();
|
ContainerId cId = createContainerId();
|
||||||
|
|
||||||
URL localResourceUri =
|
URL localResourceUri =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS.makeQualified(new Path(
|
URL.fromPath(localFS.makeQualified(new Path(
|
||||||
localResourceDir.getAbsolutePath())));
|
localResourceDir.getAbsolutePath())));
|
||||||
|
|
||||||
LocalResource localResource =
|
LocalResource localResource =
|
||||||
|
|
|
@ -741,7 +741,7 @@ public class TestNodeManagerResync {
|
||||||
ContainerLaunchContext containerLaunchContext =
|
ContainerLaunchContext containerLaunchContext =
|
||||||
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
|
|
|
@ -200,7 +200,7 @@ public class TestNodeManagerShutdown {
|
||||||
.getCanonicalHostName(), port);
|
.getCanonicalHostName(), port);
|
||||||
|
|
||||||
URL localResourceUri =
|
URL localResourceUri =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource localResource =
|
LocalResource localResource =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
|
|
|
@ -25,6 +25,7 @@ import java.io.IOException;
|
||||||
import java.net.URISyntaxException;
|
import java.net.URISyntaxException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
|
||||||
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
@ -62,8 +63,7 @@ public class TestPBRecordImpl {
|
||||||
static LocalResource createResource() {
|
static LocalResource createResource() {
|
||||||
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
assertTrue(ret instanceof LocalResourcePBImpl);
|
assertTrue(ret instanceof LocalResourcePBImpl);
|
||||||
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(
|
ret.setResource(URL.fromPath(new Path("hdfs://y.ak:8020/foo/bar")));
|
||||||
"hdfs://y.ak:8020/foo/bar")));
|
|
||||||
ret.setSize(4344L);
|
ret.setSize(4344L);
|
||||||
ret.setTimestamp(3141592653589793L);
|
ret.setTimestamp(3141592653589793L);
|
||||||
ret.setVisibility(LocalResourceVisibility.PUBLIC);
|
ret.setVisibility(LocalResourceVisibility.PUBLIC);
|
||||||
|
@ -76,7 +76,7 @@ public class TestPBRecordImpl {
|
||||||
assertTrue(ret instanceof LocalResourceStatusPBImpl);
|
assertTrue(ret instanceof LocalResourceStatusPBImpl);
|
||||||
ret.setResource(createResource());
|
ret.setResource(createResource());
|
||||||
ret.setLocalPath(
|
ret.setLocalPath(
|
||||||
ConverterUtils.getYarnUrlFromPath(
|
URL.fromPath(
|
||||||
new Path("file:///local/foo/bar")));
|
new Path("file:///local/foo/bar")));
|
||||||
ret.setStatus(ResourceStatusType.FETCH_SUCCESS);
|
ret.setStatus(ResourceStatusType.FETCH_SUCCESS);
|
||||||
ret.setLocalSize(4443L);
|
ret.setLocalSize(4443L);
|
||||||
|
@ -109,8 +109,8 @@ public class TestPBRecordImpl {
|
||||||
ResourceLocalizationSpec resource =
|
ResourceLocalizationSpec resource =
|
||||||
recordFactory.newRecordInstance(ResourceLocalizationSpec.class);
|
recordFactory.newRecordInstance(ResourceLocalizationSpec.class);
|
||||||
resource.setResource(rsrc);
|
resource.setResource(rsrc);
|
||||||
resource.setDestinationDirectory(ConverterUtils
|
resource.setDestinationDirectory(
|
||||||
.getYarnUrlFromPath(new Path("/tmp" + System.currentTimeMillis())));
|
URL.fromPath((new Path("/tmp" + System.currentTimeMillis()))));
|
||||||
rsrcs.add(resource);
|
rsrcs.add(resource);
|
||||||
ret.setResourceSpecs(rsrcs);
|
ret.setResourceSpecs(rsrcs);
|
||||||
System.out.println(resource);
|
System.out.println(resource);
|
||||||
|
|
|
@ -199,7 +199,7 @@ public class TestContainerManager extends BaseContainerManagerTest {
|
||||||
ContainerLaunchContext containerLaunchContext =
|
ContainerLaunchContext containerLaunchContext =
|
||||||
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(file.getAbsolutePath())));
|
.makeQualified(new Path(file.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
rsrc_alpha.setResource(resource_alpha);
|
rsrc_alpha.setResource(resource_alpha);
|
||||||
|
@ -229,8 +229,8 @@ public class TestContainerManager extends BaseContainerManagerTest {
|
||||||
|
|
||||||
// Now ascertain that the resources are localised correctly.
|
// Now ascertain that the resources are localised correctly.
|
||||||
ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
|
ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
|
||||||
String appIDStr = ConverterUtils.toString(appId);
|
String appIDStr = appId.toString();
|
||||||
String containerIDStr = ConverterUtils.toString(cId);
|
String containerIDStr = cId.toString();
|
||||||
File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
|
File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
|
||||||
File userDir = new File(userCacheDir, user);
|
File userDir = new File(userCacheDir, user);
|
||||||
File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
|
File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
|
||||||
|
@ -288,7 +288,7 @@ public class TestContainerManager extends BaseContainerManagerTest {
|
||||||
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
||||||
|
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
|
@ -395,7 +395,7 @@ public class TestContainerManager extends BaseContainerManagerTest {
|
||||||
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
||||||
|
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
|
@ -488,7 +488,7 @@ public class TestContainerManager extends BaseContainerManagerTest {
|
||||||
// containerLaunchContext.resources =
|
// containerLaunchContext.resources =
|
||||||
// new HashMap<CharSequence, LocalResource>();
|
// new HashMap<CharSequence, LocalResource>();
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(FileContext.getLocalFSFileContext()
|
URL.fromPath(FileContext.getLocalFSFileContext()
|
||||||
.makeQualified(new Path(file.getAbsolutePath())));
|
.makeQualified(new Path(file.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
rsrc_alpha.setResource(resource_alpha);
|
rsrc_alpha.setResource(resource_alpha);
|
||||||
|
@ -521,8 +521,8 @@ public class TestContainerManager extends BaseContainerManagerTest {
|
||||||
ApplicationState.RUNNING);
|
ApplicationState.RUNNING);
|
||||||
|
|
||||||
// Now ascertain that the resources are localised correctly.
|
// Now ascertain that the resources are localised correctly.
|
||||||
String appIDStr = ConverterUtils.toString(appId);
|
String appIDStr = appId.toString();
|
||||||
String containerIDStr = ConverterUtils.toString(cId);
|
String containerIDStr = cId.toString();
|
||||||
File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
|
File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
|
||||||
File userDir = new File(userCacheDir, user);
|
File userDir = new File(userCacheDir, user);
|
||||||
File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
|
File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
|
||||||
|
@ -975,7 +975,7 @@ public class TestContainerManager extends BaseContainerManagerTest {
|
||||||
ContainerLaunchContext containerLaunchContext =
|
ContainerLaunchContext containerLaunchContext =
|
||||||
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
|
@ -1059,7 +1059,7 @@ public class TestContainerManager extends BaseContainerManagerTest {
|
||||||
ContainerLaunchContext containerLaunchContext =
|
ContainerLaunchContext containerLaunchContext =
|
||||||
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
|
@ -1181,7 +1181,7 @@ public class TestContainerManager extends BaseContainerManagerTest {
|
||||||
ContainerId cId = createContainerId(0);
|
ContainerId cId = createContainerId(0);
|
||||||
|
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
|
|
|
@ -415,7 +415,7 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
|
||||||
fileWriter.close();
|
fileWriter.close();
|
||||||
FileContext localFS = FileContext.getLocalFSFileContext();
|
FileContext localFS = FileContext.getLocalFSFileContext();
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha = RecordFactoryProvider
|
LocalResource rsrc_alpha = RecordFactoryProvider
|
||||||
.getRecordFactory(null).newRecordInstance(LocalResource.class);
|
.getRecordFactory(null).newRecordInstance(LocalResource.class);
|
||||||
|
|
|
@ -538,7 +538,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
|
||||||
when(container.getContainerId()).thenReturn(containerId);
|
when(container.getContainerId()).thenReturn(containerId);
|
||||||
when(container.getUser()).thenReturn("test");
|
when(container.getUser()).thenReturn("test");
|
||||||
String relativeContainerLogDir = ContainerLaunch.getRelativeContainerLogDir(
|
String relativeContainerLogDir = ContainerLaunch.getRelativeContainerLogDir(
|
||||||
appId.toString(), ConverterUtils.toString(containerId));
|
appId.toString(), containerId.toString());
|
||||||
Path containerLogDir =
|
Path containerLogDir =
|
||||||
dirsHandler.getLogPathForWrite(relativeContainerLogDir, false);
|
dirsHandler.getLogPathForWrite(relativeContainerLogDir, false);
|
||||||
|
|
||||||
|
@ -744,7 +744,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
|
||||||
|
|
||||||
// upload the script file so that the container can run it
|
// upload the script file so that the container can run it
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
|
@ -945,7 +945,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
|
||||||
|
|
||||||
// upload the script file so that the container can run it
|
// upload the script file so that the container can run it
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
|
@ -1284,7 +1284,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
|
||||||
|
|
||||||
// upload the script file so that the container can run it
|
// upload the script file so that the container can run it
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
|
|
|
@ -404,7 +404,7 @@ public class TestContainerLocalizer {
|
||||||
|
|
||||||
when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
|
when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
|
||||||
when(resourceLocalizationSpec.getDestinationDirectory()).
|
when(resourceLocalizationSpec.getDestinationDirectory()).
|
||||||
thenReturn(ConverterUtils.getYarnUrlFromPath(p));
|
thenReturn(URL.fromPath(p));
|
||||||
return resourceLocalizationSpec;
|
return resourceLocalizationSpec;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -24,6 +24,7 @@ import java.util.Random;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResourceType;
|
import org.apache.hadoop.yarn.api.records.LocalResourceType;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
||||||
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest;
|
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest;
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
|
@ -39,8 +40,10 @@ public class TestLocalResource {
|
||||||
static org.apache.hadoop.yarn.api.records.LocalResource getYarnResource(Path p, long size,
|
static org.apache.hadoop.yarn.api.records.LocalResource getYarnResource(Path p, long size,
|
||||||
long timestamp, LocalResourceType type, LocalResourceVisibility state, String pattern)
|
long timestamp, LocalResourceType type, LocalResourceVisibility state, String pattern)
|
||||||
throws URISyntaxException {
|
throws URISyntaxException {
|
||||||
org.apache.hadoop.yarn.api.records.LocalResource ret = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(org.apache.hadoop.yarn.api.records.LocalResource.class);
|
org.apache.hadoop.yarn.api.records.LocalResource ret =
|
||||||
ret.setResource(ConverterUtils.getYarnUrlFromURI(p.toUri()));
|
RecordFactoryProvider.getRecordFactory(null).newRecordInstance(
|
||||||
|
org.apache.hadoop.yarn.api.records.LocalResource.class);
|
||||||
|
ret.setResource(URL.fromURI(p.toUri()));
|
||||||
ret.setSize(size);
|
ret.setSize(size);
|
||||||
ret.setTimestamp(timestamp);
|
ret.setTimestamp(timestamp);
|
||||||
ret.setType(type);
|
ret.setType(type);
|
||||||
|
|
|
@ -945,7 +945,7 @@ public class TestResourceLocalizationService {
|
||||||
// Sigh. Thread init of private localizer not accessible
|
// Sigh. Thread init of private localizer not accessible
|
||||||
Thread.sleep(1000);
|
Thread.sleep(1000);
|
||||||
dispatcher.await();
|
dispatcher.await();
|
||||||
String appStr = ConverterUtils.toString(appId);
|
String appStr = appId.toString();
|
||||||
String ctnrStr = c.getContainerId().toString();
|
String ctnrStr = c.getContainerId().toString();
|
||||||
ArgumentCaptor<LocalizerStartContext> contextCaptor = ArgumentCaptor
|
ArgumentCaptor<LocalizerStartContext> contextCaptor = ArgumentCaptor
|
||||||
.forClass(LocalizerStartContext.class);
|
.forClass(LocalizerStartContext.class);
|
||||||
|
@ -2144,12 +2144,16 @@ public class TestResourceLocalizationService {
|
||||||
// removing pending download request.
|
// removing pending download request.
|
||||||
spyService.getPublicLocalizer().pending.clear();
|
spyService.getPublicLocalizer().pending.clear();
|
||||||
|
|
||||||
|
LocalizerContext lc = mock(LocalizerContext.class);
|
||||||
|
when(lc.getContainerId()).thenReturn(ContainerId.newContainerId(
|
||||||
|
ApplicationAttemptId.newInstance(ApplicationId.newInstance(1L, 1), 1),
|
||||||
|
1L));
|
||||||
|
|
||||||
// Now I need to simulate a race condition wherein Event is added to
|
// Now I need to simulate a race condition wherein Event is added to
|
||||||
// dispatcher before resource state changes to either FAILED or LOCALIZED
|
// dispatcher before resource state changes to either FAILED or LOCALIZED
|
||||||
// Hence sending event directly to dispatcher.
|
// Hence sending event directly to dispatcher.
|
||||||
LocalizerResourceRequestEvent localizerEvent =
|
LocalizerResourceRequestEvent localizerEvent =
|
||||||
new LocalizerResourceRequestEvent(lr, null,
|
new LocalizerResourceRequestEvent(lr, null, lc, null);
|
||||||
mock(LocalizerContext.class), null);
|
|
||||||
|
|
||||||
dispatcher1.getEventHandler().handle(localizerEvent);
|
dispatcher1.getEventHandler().handle(localizerEvent);
|
||||||
// Waiting for download to start. This should return false as new download
|
// Waiting for download to start. This should return false as new download
|
||||||
|
@ -2457,7 +2461,7 @@ public class TestResourceLocalizationService {
|
||||||
BuilderUtils.newApplicationId(314159265358979L, 3);
|
BuilderUtils.newApplicationId(314159265358979L, 3);
|
||||||
when(app.getUser()).thenReturn(user);
|
when(app.getUser()).thenReturn(user);
|
||||||
when(app.getAppId()).thenReturn(appId);
|
when(app.getAppId()).thenReturn(appId);
|
||||||
when(app.toString()).thenReturn(ConverterUtils.toString(appId));
|
when(app.toString()).thenReturn(appId.toString());
|
||||||
|
|
||||||
// init container.
|
// init container.
|
||||||
final Container c = getMockContainer(appId, 42, user);
|
final Container c = getMockContainer(appId, 42, user);
|
||||||
|
@ -2468,17 +2472,16 @@ public class TestResourceLocalizationService {
|
||||||
Path usersdir = new Path(tmpDirs.get(i), ContainerLocalizer.USERCACHE);
|
Path usersdir = new Path(tmpDirs.get(i), ContainerLocalizer.USERCACHE);
|
||||||
Path userdir = new Path(usersdir, user);
|
Path userdir = new Path(usersdir, user);
|
||||||
Path allAppsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
|
Path allAppsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
|
||||||
Path appDir = new Path(allAppsdir, ConverterUtils.toString(appId));
|
Path appDir = new Path(allAppsdir, appId.toString());
|
||||||
Path containerDir =
|
Path containerDir =
|
||||||
new Path(appDir, ConverterUtils.toString(c.getContainerId()));
|
new Path(appDir, c.getContainerId().toString());
|
||||||
containerLocalDirs.add(containerDir);
|
containerLocalDirs.add(containerDir);
|
||||||
appLocalDirs.add(appDir);
|
appLocalDirs.add(appDir);
|
||||||
|
|
||||||
Path sysDir =
|
Path sysDir =
|
||||||
new Path(tmpDirs.get(i), ResourceLocalizationService.NM_PRIVATE_DIR);
|
new Path(tmpDirs.get(i), ResourceLocalizationService.NM_PRIVATE_DIR);
|
||||||
Path appSysDir = new Path(sysDir, ConverterUtils.toString(appId));
|
Path appSysDir = new Path(sysDir, appId.toString());
|
||||||
Path containerSysDir =
|
Path containerSysDir = new Path(appSysDir, c.getContainerId().toString());
|
||||||
new Path(appSysDir, ConverterUtils.toString(c.getContainerId()));
|
|
||||||
|
|
||||||
nmLocalContainerDirs.add(containerSysDir);
|
nmLocalContainerDirs.add(containerSysDir);
|
||||||
nmLocalAppDirs.add(appSysDir);
|
nmLocalAppDirs.add(appSysDir);
|
||||||
|
|
|
@ -108,10 +108,8 @@ public class TestAppLogAggregatorImpl {
|
||||||
final ContainerId containerId = ContainerId.newContainerId(attemptId, 0);
|
final ContainerId containerId = ContainerId.newContainerId(attemptId, 0);
|
||||||
|
|
||||||
// create artificial log files
|
// create artificial log files
|
||||||
final File appLogDir = new File(LOCAL_LOG_DIR,
|
final File appLogDir = new File(LOCAL_LOG_DIR, applicationId.toString());
|
||||||
ConverterUtils.toString(applicationId));
|
final File containerLogDir = new File(appLogDir, containerId.toString());
|
||||||
final File containerLogDir = new File(appLogDir,
|
|
||||||
ConverterUtils.toString(containerId));
|
|
||||||
containerLogDir.mkdirs();
|
containerLogDir.mkdirs();
|
||||||
final Set<File> logFiles = createContainerLogFiles(containerLogDir, 3);
|
final Set<File> logFiles = createContainerLogFiles(containerLogDir, 3);
|
||||||
|
|
||||||
|
@ -135,9 +133,9 @@ public class TestAppLogAggregatorImpl {
|
||||||
|
|
||||||
// create artificial log files
|
// create artificial log files
|
||||||
final File appLogDir = new File(LOCAL_LOG_DIR,
|
final File appLogDir = new File(LOCAL_LOG_DIR,
|
||||||
ConverterUtils.toString(applicationId));
|
applicationId.toString());
|
||||||
final File containerLogDir = new File(appLogDir,
|
final File containerLogDir = new File(appLogDir,
|
||||||
ConverterUtils.toString(containerId));
|
containerId.toString());
|
||||||
containerLogDir.mkdirs();
|
containerLogDir.mkdirs();
|
||||||
final Set<File> logFiles = createContainerLogFiles(containerLogDir, 3);
|
final Set<File> logFiles = createContainerLogFiles(containerLogDir, 3);
|
||||||
|
|
||||||
|
@ -163,9 +161,9 @@ public class TestAppLogAggregatorImpl {
|
||||||
|
|
||||||
// create artificial log files
|
// create artificial log files
|
||||||
final File appLogDir = new File(LOCAL_LOG_DIR,
|
final File appLogDir = new File(LOCAL_LOG_DIR,
|
||||||
ConverterUtils.toString(applicationId));
|
applicationId.toString());
|
||||||
final File containerLogDir = new File(appLogDir,
|
final File containerLogDir = new File(appLogDir,
|
||||||
ConverterUtils.toString(containerId));
|
containerId.toString());
|
||||||
containerLogDir.mkdirs();
|
containerLogDir.mkdirs();
|
||||||
final Set<File> logFiles = createContainerLogFiles(containerLogDir, 3);
|
final Set<File> logFiles = createContainerLogFiles(containerLogDir, 3);
|
||||||
|
|
||||||
|
|
|
@ -194,7 +194,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
|
||||||
|
|
||||||
// AppLogDir should be created
|
// AppLogDir should be created
|
||||||
File app1LogDir =
|
File app1LogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(application1));
|
new File(localLogDir, application1.toString());
|
||||||
app1LogDir.mkdir();
|
app1LogDir.mkdir();
|
||||||
logAggregationService
|
logAggregationService
|
||||||
.handle(new LogHandlerAppStartedEvent(
|
.handle(new LogHandlerAppStartedEvent(
|
||||||
|
@ -221,7 +221,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
|
||||||
verify(delSrvc).delete(eq(user), eq((Path) null),
|
verify(delSrvc).delete(eq(user), eq((Path) null),
|
||||||
eq(new Path(app1LogDir.getAbsolutePath())));
|
eq(new Path(app1LogDir.getAbsolutePath())));
|
||||||
|
|
||||||
String containerIdStr = ConverterUtils.toString(container11);
|
String containerIdStr = container11.toString();
|
||||||
File containerLogDir = new File(app1LogDir, containerIdStr);
|
File containerLogDir = new File(app1LogDir, containerIdStr);
|
||||||
int count = 0;
|
int count = 0;
|
||||||
int maxAttempts = 50;
|
int maxAttempts = 50;
|
||||||
|
@ -315,7 +315,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
|
||||||
logAggregationService.start();
|
logAggregationService.start();
|
||||||
|
|
||||||
ApplicationId app = BuilderUtils.newApplicationId(1234, 1);
|
ApplicationId app = BuilderUtils.newApplicationId(1234, 1);
|
||||||
File appLogDir = new File(localLogDir, ConverterUtils.toString(app));
|
File appLogDir = new File(localLogDir, app.toString());
|
||||||
appLogDir.mkdir();
|
appLogDir.mkdir();
|
||||||
LogAggregationContext context =
|
LogAggregationContext context =
|
||||||
LogAggregationContext.newInstance("HOST*", "sys*");
|
LogAggregationContext.newInstance("HOST*", "sys*");
|
||||||
|
@ -352,7 +352,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
|
||||||
|
|
||||||
// AppLogDir should be created
|
// AppLogDir should be created
|
||||||
File app1LogDir =
|
File app1LogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(application1));
|
new File(localLogDir, application1.toString());
|
||||||
app1LogDir.mkdir();
|
app1LogDir.mkdir();
|
||||||
logAggregationService
|
logAggregationService
|
||||||
.handle(new LogHandlerAppStartedEvent(
|
.handle(new LogHandlerAppStartedEvent(
|
||||||
|
@ -402,7 +402,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
|
||||||
|
|
||||||
// AppLogDir should be created
|
// AppLogDir should be created
|
||||||
File app1LogDir =
|
File app1LogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(application1));
|
new File(localLogDir, application1.toString());
|
||||||
app1LogDir.mkdir();
|
app1LogDir.mkdir();
|
||||||
logAggregationService
|
logAggregationService
|
||||||
.handle(new LogHandlerAppStartedEvent(
|
.handle(new LogHandlerAppStartedEvent(
|
||||||
|
@ -423,7 +423,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
|
||||||
BuilderUtils.newApplicationAttemptId(application2, 1);
|
BuilderUtils.newApplicationAttemptId(application2, 1);
|
||||||
|
|
||||||
File app2LogDir =
|
File app2LogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(application2));
|
new File(localLogDir, application2.toString());
|
||||||
app2LogDir.mkdir();
|
app2LogDir.mkdir();
|
||||||
LogAggregationContext contextWithAMOnly =
|
LogAggregationContext contextWithAMOnly =
|
||||||
Records.newRecord(LogAggregationContext.class);
|
Records.newRecord(LogAggregationContext.class);
|
||||||
|
@ -452,7 +452,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
|
||||||
BuilderUtils.newApplicationAttemptId(application3, 1);
|
BuilderUtils.newApplicationAttemptId(application3, 1);
|
||||||
|
|
||||||
File app3LogDir =
|
File app3LogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(application3));
|
new File(localLogDir, application3.toString());
|
||||||
app3LogDir.mkdir();
|
app3LogDir.mkdir();
|
||||||
LogAggregationContext contextWithAMAndFailed =
|
LogAggregationContext contextWithAMAndFailed =
|
||||||
Records.newRecord(LogAggregationContext.class);
|
Records.newRecord(LogAggregationContext.class);
|
||||||
|
@ -583,7 +583,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
|
||||||
BuilderUtils.newApplicationId(System.currentTimeMillis(),
|
BuilderUtils.newApplicationId(System.currentTimeMillis(),
|
||||||
(int) (Math.random() * 1000));
|
(int) (Math.random() * 1000));
|
||||||
File appLogDir =
|
File appLogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(appId2));
|
new File(localLogDir, appId2.toString());
|
||||||
appLogDir.mkdir();
|
appLogDir.mkdir();
|
||||||
logAggregationService.handle(new LogHandlerAppStartedEvent(appId2,
|
logAggregationService.handle(new LogHandlerAppStartedEvent(appId2,
|
||||||
this.user, null, this.acls, contextWithAMAndFailed));
|
this.user, null, this.acls, contextWithAMAndFailed));
|
||||||
|
@ -758,7 +758,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
|
||||||
(int) (Math.random() * 1000));
|
(int) (Math.random() * 1000));
|
||||||
|
|
||||||
File appLogDir =
|
File appLogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(appId));
|
new File(localLogDir, appId.toString());
|
||||||
appLogDir.mkdir();
|
appLogDir.mkdir();
|
||||||
|
|
||||||
Exception e = new RuntimeException("KABOOM!");
|
Exception e = new RuntimeException("KABOOM!");
|
||||||
|
@ -805,7 +805,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
|
||||||
private void writeContainerLogs(File appLogDir, ContainerId containerId,
|
private void writeContainerLogs(File appLogDir, ContainerId containerId,
|
||||||
String[] fileName) throws IOException {
|
String[] fileName) throws IOException {
|
||||||
// ContainerLogDir should be created
|
// ContainerLogDir should be created
|
||||||
String containerStr = ConverterUtils.toString(containerId);
|
String containerStr = containerId.toString();
|
||||||
File containerLogDir = new File(appLogDir, containerStr);
|
File containerLogDir = new File(appLogDir, containerStr);
|
||||||
boolean created = containerLogDir.mkdirs();
|
boolean created = containerLogDir.mkdirs();
|
||||||
LOG.info("Created Dir:" + containerLogDir.getAbsolutePath() + " status :"
|
LOG.info("Created Dir:" + containerLogDir.getAbsolutePath() + " status :"
|
||||||
|
@ -943,7 +943,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
|
||||||
Assert.assertTrue("number of containers with logs should be at most " +
|
Assert.assertTrue("number of containers with logs should be at most " +
|
||||||
minNumOfContainers,logMap.size() <= maxNumOfContainers);
|
minNumOfContainers,logMap.size() <= maxNumOfContainers);
|
||||||
for (ContainerId cId : expectedContainerIds) {
|
for (ContainerId cId : expectedContainerIds) {
|
||||||
String containerStr = ConverterUtils.toString(cId);
|
String containerStr = cId.toString();
|
||||||
Map<String, String> thisContainerMap = logMap.remove(containerStr);
|
Map<String, String> thisContainerMap = logMap.remove(containerStr);
|
||||||
Assert.assertEquals(numOfLogsPerContainer, thisContainerMap.size());
|
Assert.assertEquals(numOfLogsPerContainer, thisContainerMap.size());
|
||||||
for (String fileType : logFiles) {
|
for (String fileType : logFiles) {
|
||||||
|
@ -998,7 +998,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
|
||||||
ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0);
|
ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0);
|
||||||
|
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
|
@ -1435,7 +1435,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
|
||||||
// has only logs from stdout and syslog
|
// has only logs from stdout and syslog
|
||||||
// AppLogDir should be created
|
// AppLogDir should be created
|
||||||
File appLogDir1 =
|
File appLogDir1 =
|
||||||
new File(localLogDir, ConverterUtils.toString(application1));
|
new File(localLogDir, application1.toString());
|
||||||
appLogDir1.mkdir();
|
appLogDir1.mkdir();
|
||||||
logAggregationService.handle(new LogHandlerAppStartedEvent(application1,
|
logAggregationService.handle(new LogHandlerAppStartedEvent(application1,
|
||||||
this.user, null, this.acls,
|
this.user, null, this.acls,
|
||||||
|
@ -1460,7 +1460,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
|
||||||
BuilderUtils.newApplicationAttemptId(application2, 1);
|
BuilderUtils.newApplicationAttemptId(application2, 1);
|
||||||
|
|
||||||
File app2LogDir =
|
File app2LogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(application2));
|
new File(localLogDir, application2.toString());
|
||||||
app2LogDir.mkdir();
|
app2LogDir.mkdir();
|
||||||
LogAggregationContextWithExcludePatterns.setLogAggregationPolicyClassName(
|
LogAggregationContextWithExcludePatterns.setLogAggregationPolicyClassName(
|
||||||
AMOnlyLogAggregationPolicy.class.getName());
|
AMOnlyLogAggregationPolicy.class.getName());
|
||||||
|
@ -1485,7 +1485,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
|
||||||
ApplicationAttemptId appAttemptId3 =
|
ApplicationAttemptId appAttemptId3 =
|
||||||
BuilderUtils.newApplicationAttemptId(application3, 1);
|
BuilderUtils.newApplicationAttemptId(application3, 1);
|
||||||
File app3LogDir =
|
File app3LogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(application3));
|
new File(localLogDir, application3.toString());
|
||||||
app3LogDir.mkdir();
|
app3LogDir.mkdir();
|
||||||
context1.setLogAggregationPolicyClassName(
|
context1.setLogAggregationPolicyClassName(
|
||||||
AMOnlyLogAggregationPolicy.class.getName());
|
AMOnlyLogAggregationPolicy.class.getName());
|
||||||
|
@ -1510,7 +1510,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
|
||||||
ApplicationAttemptId appAttemptId4 =
|
ApplicationAttemptId appAttemptId4 =
|
||||||
BuilderUtils.newApplicationAttemptId(application4, 1);
|
BuilderUtils.newApplicationAttemptId(application4, 1);
|
||||||
File app4LogDir =
|
File app4LogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(application4));
|
new File(localLogDir, application4.toString());
|
||||||
app4LogDir.mkdir();
|
app4LogDir.mkdir();
|
||||||
context2.setLogAggregationPolicyClassName(
|
context2.setLogAggregationPolicyClassName(
|
||||||
AMOnlyLogAggregationPolicy.class.getName());
|
AMOnlyLogAggregationPolicy.class.getName());
|
||||||
|
@ -2012,7 +2012,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
|
||||||
containerType);
|
containerType);
|
||||||
// Simulate log-file creation
|
// Simulate log-file creation
|
||||||
File appLogDir1 =
|
File appLogDir1 =
|
||||||
new File(localLogDir, ConverterUtils.toString(application1));
|
new File(localLogDir, application1.toString());
|
||||||
appLogDir1.mkdir();
|
appLogDir1.mkdir();
|
||||||
writeContainerLogs(appLogDir1, containerId, logFiles);
|
writeContainerLogs(appLogDir1, containerId, logFiles);
|
||||||
|
|
||||||
|
@ -2123,7 +2123,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
|
||||||
|
|
||||||
// AppLogDir should be created
|
// AppLogDir should be created
|
||||||
File appLogDir =
|
File appLogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(application));
|
new File(localLogDir, application.toString());
|
||||||
appLogDir.mkdir();
|
appLogDir.mkdir();
|
||||||
logAggregationService.handle(new LogHandlerAppStartedEvent(application,
|
logAggregationService.handle(new LogHandlerAppStartedEvent(application,
|
||||||
this.user, null, this.acls, logAggregationContextWithInterval));
|
this.user, null, this.acls, logAggregationContextWithInterval));
|
||||||
|
|
|
@ -210,7 +210,7 @@ public class TestContainersMonitor extends BaseContainerManagerTest {
|
||||||
ContainerId cId = ContainerId.newContainerId(appAttemptId, 0);
|
ContainerId cId = ContainerId.newContainerId(appAttemptId, 0);
|
||||||
|
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
|
|
|
@ -374,7 +374,7 @@ public class TestNMLeveldbStateStoreService {
|
||||||
Path appRsrcPath = new Path("hdfs://some/app/resource");
|
Path appRsrcPath = new Path("hdfs://some/app/resource");
|
||||||
LocalResourcePBImpl rsrcPb = (LocalResourcePBImpl)
|
LocalResourcePBImpl rsrcPb = (LocalResourcePBImpl)
|
||||||
LocalResource.newInstance(
|
LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(appRsrcPath),
|
URL.fromPath(appRsrcPath),
|
||||||
LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION,
|
LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION,
|
||||||
123L, 456L);
|
123L, 456L);
|
||||||
LocalResourceProto appRsrcProto = rsrcPb.getProto();
|
LocalResourceProto appRsrcProto = rsrcPb.getProto();
|
||||||
|
@ -407,7 +407,7 @@ public class TestNMLeveldbStateStoreService {
|
||||||
// start some public and private resources
|
// start some public and private resources
|
||||||
Path pubRsrcPath1 = new Path("hdfs://some/public/resource1");
|
Path pubRsrcPath1 = new Path("hdfs://some/public/resource1");
|
||||||
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(pubRsrcPath1),
|
URL.fromPath(pubRsrcPath1),
|
||||||
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
||||||
789L, 135L);
|
789L, 135L);
|
||||||
LocalResourceProto pubRsrcProto1 = rsrcPb.getProto();
|
LocalResourceProto pubRsrcProto1 = rsrcPb.getProto();
|
||||||
|
@ -416,7 +416,7 @@ public class TestNMLeveldbStateStoreService {
|
||||||
pubRsrcLocalPath1);
|
pubRsrcLocalPath1);
|
||||||
Path pubRsrcPath2 = new Path("hdfs://some/public/resource2");
|
Path pubRsrcPath2 = new Path("hdfs://some/public/resource2");
|
||||||
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(pubRsrcPath2),
|
URL.fromPath(pubRsrcPath2),
|
||||||
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
||||||
789L, 135L);
|
789L, 135L);
|
||||||
LocalResourceProto pubRsrcProto2 = rsrcPb.getProto();
|
LocalResourceProto pubRsrcProto2 = rsrcPb.getProto();
|
||||||
|
@ -425,7 +425,7 @@ public class TestNMLeveldbStateStoreService {
|
||||||
pubRsrcLocalPath2);
|
pubRsrcLocalPath2);
|
||||||
Path privRsrcPath = new Path("hdfs://some/private/resource");
|
Path privRsrcPath = new Path("hdfs://some/private/resource");
|
||||||
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(privRsrcPath),
|
URL.fromPath(privRsrcPath),
|
||||||
LocalResourceType.PATTERN, LocalResourceVisibility.PRIVATE,
|
LocalResourceType.PATTERN, LocalResourceVisibility.PRIVATE,
|
||||||
789L, 680L, "*pattern*");
|
789L, 680L, "*pattern*");
|
||||||
LocalResourceProto privRsrcProto = rsrcPb.getProto();
|
LocalResourceProto privRsrcProto = rsrcPb.getProto();
|
||||||
|
@ -470,7 +470,7 @@ public class TestNMLeveldbStateStoreService {
|
||||||
Path appRsrcPath = new Path("hdfs://some/app/resource");
|
Path appRsrcPath = new Path("hdfs://some/app/resource");
|
||||||
LocalResourcePBImpl rsrcPb = (LocalResourcePBImpl)
|
LocalResourcePBImpl rsrcPb = (LocalResourcePBImpl)
|
||||||
LocalResource.newInstance(
|
LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(appRsrcPath),
|
URL.fromPath(appRsrcPath),
|
||||||
LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION,
|
LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION,
|
||||||
123L, 456L);
|
123L, 456L);
|
||||||
LocalResourceProto appRsrcProto = rsrcPb.getProto();
|
LocalResourceProto appRsrcProto = rsrcPb.getProto();
|
||||||
|
@ -510,7 +510,7 @@ public class TestNMLeveldbStateStoreService {
|
||||||
// start some public and private resources
|
// start some public and private resources
|
||||||
Path pubRsrcPath1 = new Path("hdfs://some/public/resource1");
|
Path pubRsrcPath1 = new Path("hdfs://some/public/resource1");
|
||||||
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(pubRsrcPath1),
|
URL.fromPath(pubRsrcPath1),
|
||||||
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
||||||
789L, 135L);
|
789L, 135L);
|
||||||
LocalResourceProto pubRsrcProto1 = rsrcPb.getProto();
|
LocalResourceProto pubRsrcProto1 = rsrcPb.getProto();
|
||||||
|
@ -519,7 +519,7 @@ public class TestNMLeveldbStateStoreService {
|
||||||
pubRsrcLocalPath1);
|
pubRsrcLocalPath1);
|
||||||
Path pubRsrcPath2 = new Path("hdfs://some/public/resource2");
|
Path pubRsrcPath2 = new Path("hdfs://some/public/resource2");
|
||||||
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(pubRsrcPath2),
|
URL.fromPath(pubRsrcPath2),
|
||||||
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
||||||
789L, 135L);
|
789L, 135L);
|
||||||
LocalResourceProto pubRsrcProto2 = rsrcPb.getProto();
|
LocalResourceProto pubRsrcProto2 = rsrcPb.getProto();
|
||||||
|
@ -528,7 +528,7 @@ public class TestNMLeveldbStateStoreService {
|
||||||
pubRsrcLocalPath2);
|
pubRsrcLocalPath2);
|
||||||
Path privRsrcPath = new Path("hdfs://some/private/resource");
|
Path privRsrcPath = new Path("hdfs://some/private/resource");
|
||||||
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(privRsrcPath),
|
URL.fromPath(privRsrcPath),
|
||||||
LocalResourceType.PATTERN, LocalResourceVisibility.PRIVATE,
|
LocalResourceType.PATTERN, LocalResourceVisibility.PRIVATE,
|
||||||
789L, 680L, "*pattern*");
|
789L, 680L, "*pattern*");
|
||||||
LocalResourceProto privRsrcProto = rsrcPb.getProto();
|
LocalResourceProto privRsrcProto = rsrcPb.getProto();
|
||||||
|
@ -589,7 +589,7 @@ public class TestNMLeveldbStateStoreService {
|
||||||
Path appRsrcPath = new Path("hdfs://some/app/resource");
|
Path appRsrcPath = new Path("hdfs://some/app/resource");
|
||||||
LocalResourcePBImpl rsrcPb = (LocalResourcePBImpl)
|
LocalResourcePBImpl rsrcPb = (LocalResourcePBImpl)
|
||||||
LocalResource.newInstance(
|
LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(appRsrcPath),
|
URL.fromPath(appRsrcPath),
|
||||||
LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION,
|
LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION,
|
||||||
123L, 456L);
|
123L, 456L);
|
||||||
LocalResourceProto appRsrcProto = rsrcPb.getProto();
|
LocalResourceProto appRsrcProto = rsrcPb.getProto();
|
||||||
|
@ -619,7 +619,7 @@ public class TestNMLeveldbStateStoreService {
|
||||||
// add public and private resources and remove some
|
// add public and private resources and remove some
|
||||||
Path pubRsrcPath1 = new Path("hdfs://some/public/resource1");
|
Path pubRsrcPath1 = new Path("hdfs://some/public/resource1");
|
||||||
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(pubRsrcPath1),
|
URL.fromPath(pubRsrcPath1),
|
||||||
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
||||||
789L, 135L);
|
789L, 135L);
|
||||||
LocalResourceProto pubRsrcProto1 = rsrcPb.getProto();
|
LocalResourceProto pubRsrcProto1 = rsrcPb.getProto();
|
||||||
|
@ -635,7 +635,7 @@ public class TestNMLeveldbStateStoreService {
|
||||||
stateStore.finishResourceLocalization(null, null, pubLocalizedProto1);
|
stateStore.finishResourceLocalization(null, null, pubLocalizedProto1);
|
||||||
Path pubRsrcPath2 = new Path("hdfs://some/public/resource2");
|
Path pubRsrcPath2 = new Path("hdfs://some/public/resource2");
|
||||||
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(pubRsrcPath2),
|
URL.fromPath(pubRsrcPath2),
|
||||||
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
||||||
789L, 135L);
|
789L, 135L);
|
||||||
LocalResourceProto pubRsrcProto2 = rsrcPb.getProto();
|
LocalResourceProto pubRsrcProto2 = rsrcPb.getProto();
|
||||||
|
@ -652,7 +652,7 @@ public class TestNMLeveldbStateStoreService {
|
||||||
stateStore.removeLocalizedResource(null, null, pubRsrcLocalPath2);
|
stateStore.removeLocalizedResource(null, null, pubRsrcLocalPath2);
|
||||||
Path privRsrcPath = new Path("hdfs://some/private/resource");
|
Path privRsrcPath = new Path("hdfs://some/private/resource");
|
||||||
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(privRsrcPath),
|
URL.fromPath(privRsrcPath),
|
||||||
LocalResourceType.PATTERN, LocalResourceVisibility.PRIVATE,
|
LocalResourceType.PATTERN, LocalResourceVisibility.PRIVATE,
|
||||||
789L, 680L, "*pattern*");
|
789L, 680L, "*pattern*");
|
||||||
LocalResourceProto privRsrcProto = rsrcPb.getProto();
|
LocalResourceProto privRsrcProto = rsrcPb.getProto();
|
||||||
|
|
|
@ -249,7 +249,7 @@ public class TestNMWebServer {
|
||||||
containerLogDir.mkdirs();
|
containerLogDir.mkdirs();
|
||||||
for (String fileType : new String[] { "stdout", "stderr", "syslog" }) {
|
for (String fileType : new String[] { "stdout", "stderr", "syslog" }) {
|
||||||
Writer writer = new FileWriter(new File(containerLogDir, fileType));
|
Writer writer = new FileWriter(new File(containerLogDir, fileType));
|
||||||
writer.write(ConverterUtils.toString(containerId) + "\n Hello "
|
writer.write(containerId.toString() + "\n Hello "
|
||||||
+ fileType + "!");
|
+ fileType + "!");
|
||||||
writer.close();
|
writer.close();
|
||||||
}
|
}
|
||||||
|
|
|
@ -40,6 +40,7 @@ import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.FileUtil;
|
import org.apache.hadoop.fs.FileUtil;
|
||||||
import org.apache.hadoop.util.NodeHealthScriptRunner;
|
import org.apache.hadoop.util.NodeHealthScriptRunner;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.event.AsyncDispatcher;
|
import org.apache.hadoop.yarn.event.AsyncDispatcher;
|
||||||
|
@ -280,7 +281,7 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
|
||||||
verifyNodeContainerInfo(
|
verifyNodeContainerInfo(
|
||||||
conInfo.getJSONObject(i),
|
conInfo.getJSONObject(i),
|
||||||
nmContext.getContainers().get(
|
nmContext.getContainers().get(
|
||||||
ConverterUtils.toContainerId(conInfo.getJSONObject(i).getString(
|
ContainerId.fromString(conInfo.getJSONObject(i).getString(
|
||||||
"id"))));
|
"id"))));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -316,7 +317,7 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
|
||||||
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
|
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
|
||||||
JSONObject json = response.getEntity(JSONObject.class);
|
JSONObject json = response.getEntity(JSONObject.class);
|
||||||
verifyNodeContainerInfo(json.getJSONObject("container"), nmContext
|
verifyNodeContainerInfo(json.getJSONObject("container"), nmContext
|
||||||
.getContainers().get(ConverterUtils.toContainerId(id)));
|
.getContainers().get(ContainerId.fromString(id)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -449,7 +450,7 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
|
||||||
NodeList nodes = dom.getElementsByTagName("container");
|
NodeList nodes = dom.getElementsByTagName("container");
|
||||||
assertEquals("incorrect number of elements", 1, nodes.getLength());
|
assertEquals("incorrect number of elements", 1, nodes.getLength());
|
||||||
verifyContainersInfoXML(nodes,
|
verifyContainersInfoXML(nodes,
|
||||||
nmContext.getContainers().get(ConverterUtils.toContainerId(id)));
|
nmContext.getContainers().get(ContainerId.fromString(id)));
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1345,7 +1345,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
|
||||||
rmStore.init(conf);
|
rmStore.init(conf);
|
||||||
rmStore.start();
|
rmStore.start();
|
||||||
try {
|
try {
|
||||||
ApplicationId removeAppId = ConverterUtils.toApplicationId(applicationId);
|
ApplicationId removeAppId = ApplicationId.fromString(applicationId);
|
||||||
LOG.info("Deleting application " + removeAppId + " from state store");
|
LOG.info("Deleting application " + removeAppId + " from state store");
|
||||||
rmStore.removeApplication(removeAppId);
|
rmStore.removeApplication(removeAppId);
|
||||||
LOG.info("Application is deleted from state store");
|
LOG.info("Application is deleted from state store");
|
||||||
|
|
|
@ -499,7 +499,7 @@ public class LeveldbRMStateStore extends RMStateStore {
|
||||||
|
|
||||||
private ApplicationStateData createApplicationState(String appIdStr,
|
private ApplicationStateData createApplicationState(String appIdStr,
|
||||||
byte[] data) throws IOException {
|
byte[] data) throws IOException {
|
||||||
ApplicationId appId = ConverterUtils.toApplicationId(appIdStr);
|
ApplicationId appId = ApplicationId.fromString(appIdStr);
|
||||||
ApplicationStateDataPBImpl appState =
|
ApplicationStateDataPBImpl appState =
|
||||||
new ApplicationStateDataPBImpl(
|
new ApplicationStateDataPBImpl(
|
||||||
ApplicationStateDataProto.parseFrom(data));
|
ApplicationStateDataProto.parseFrom(data));
|
||||||
|
@ -545,8 +545,7 @@ public class LeveldbRMStateStore extends RMStateStore {
|
||||||
|
|
||||||
private ApplicationAttemptStateData createAttemptState(String itemName,
|
private ApplicationAttemptStateData createAttemptState(String itemName,
|
||||||
byte[] data) throws IOException {
|
byte[] data) throws IOException {
|
||||||
ApplicationAttemptId attemptId =
|
ApplicationAttemptId attemptId = ApplicationAttemptId.fromString(itemName);
|
||||||
ConverterUtils.toApplicationAttemptId(itemName);
|
|
||||||
ApplicationAttemptStateDataPBImpl attemptState =
|
ApplicationAttemptStateDataPBImpl attemptState =
|
||||||
new ApplicationAttemptStateDataPBImpl(
|
new ApplicationAttemptStateDataPBImpl(
|
||||||
ApplicationAttemptStateDataProto.parseFrom(data));
|
ApplicationAttemptStateDataProto.parseFrom(data));
|
||||||
|
|
|
@ -514,7 +514,7 @@ public class ZKRMStateStore extends RMStateStore {
|
||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
LOG.debug("Loading application from znode: " + childNodeName);
|
LOG.debug("Loading application from znode: " + childNodeName);
|
||||||
}
|
}
|
||||||
ApplicationId appId = ConverterUtils.toApplicationId(childNodeName);
|
ApplicationId appId = ApplicationId.fromString(childNodeName);
|
||||||
ApplicationStateDataPBImpl appState =
|
ApplicationStateDataPBImpl appState =
|
||||||
new ApplicationStateDataPBImpl(
|
new ApplicationStateDataPBImpl(
|
||||||
ApplicationStateDataProto.parseFrom(childData));
|
ApplicationStateDataProto.parseFrom(childData));
|
||||||
|
|
|
@ -133,7 +133,7 @@ public class DynamicResourceConfiguration extends Configuration {
|
||||||
= new HashMap<NodeId, ResourceOption> ();
|
= new HashMap<NodeId, ResourceOption> ();
|
||||||
|
|
||||||
for (String node : nodes) {
|
for (String node : nodes) {
|
||||||
NodeId nid = ConverterUtils.toNodeId(node);
|
NodeId nid = NodeId.fromString(node);
|
||||||
int vcores = getVcoresPerNode(node);
|
int vcores = getVcoresPerNode(node);
|
||||||
int memory = getMemoryPerNode(node);
|
int memory = getMemoryPerNode(node);
|
||||||
int overCommitTimeout = getOverCommitTimeoutPerNode(node);
|
int overCommitTimeout = getOverCommitTimeoutPerNode(node);
|
||||||
|
|
|
@ -346,7 +346,7 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
|
||||||
logURL.append(WebAppUtils.getHttpSchemePrefix(rmContext
|
logURL.append(WebAppUtils.getHttpSchemePrefix(rmContext
|
||||||
.getYarnConfiguration()));
|
.getYarnConfiguration()));
|
||||||
logURL.append(WebAppUtils.getRunningLogURL(
|
logURL.append(WebAppUtils.getRunningLogURL(
|
||||||
container.getNodeHttpAddress(), ConverterUtils.toString(containerId),
|
container.getNodeHttpAddress(), containerId.toString(),
|
||||||
user));
|
user));
|
||||||
return logURL.toString();
|
return logURL.toString();
|
||||||
} finally {
|
} finally {
|
||||||
|
|
|
@ -83,8 +83,8 @@ public class RMAppsBlock extends AppsBlock {
|
||||||
}
|
}
|
||||||
|
|
||||||
AppInfo app = new AppInfo(appReport);
|
AppInfo app = new AppInfo(appReport);
|
||||||
ApplicationAttemptId appAttemptId =
|
ApplicationAttemptId appAttemptId = ApplicationAttemptId.fromString(
|
||||||
ConverterUtils.toApplicationAttemptId(app.getCurrentAppAttemptId());
|
app.getCurrentAppAttemptId());
|
||||||
String queuePercent = "N/A";
|
String queuePercent = "N/A";
|
||||||
String clusterPercent = "N/A";
|
String clusterPercent = "N/A";
|
||||||
if(appReport.getApplicationResourceUsageReport() != null) {
|
if(appReport.getApplicationResourceUsageReport() != null) {
|
||||||
|
|
|
@ -220,7 +220,7 @@ public class RMWebAppFilter extends GuiceContainer {
|
||||||
break;
|
break;
|
||||||
case "appattempt":
|
case "appattempt":
|
||||||
try{
|
try{
|
||||||
appAttemptId = ConverterUtils.toApplicationAttemptId(parts[3]);
|
appAttemptId = ApplicationAttemptId.fromString(parts[3]);
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
LOG.debug("Error parsing {} as an ApplicationAttemptId",
|
LOG.debug("Error parsing {} as an ApplicationAttemptId",
|
||||||
parts[3], e);
|
parts[3], e);
|
||||||
|
|
|
@ -113,6 +113,7 @@ import org.apache.hadoop.yarn.api.records.ReservationRequest;
|
||||||
import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
|
import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
|
||||||
import org.apache.hadoop.yarn.api.records.ReservationRequests;
|
import org.apache.hadoop.yarn.api.records.ReservationRequests;
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
|
@ -373,7 +374,7 @@ public class RMWebServices extends WebServices {
|
||||||
if (sched == null) {
|
if (sched == null) {
|
||||||
throw new NotFoundException("Null ResourceScheduler instance");
|
throw new NotFoundException("Null ResourceScheduler instance");
|
||||||
}
|
}
|
||||||
NodeId nid = ConverterUtils.toNodeId(nodeId);
|
NodeId nid = NodeId.fromString(nodeId);
|
||||||
RMNode ni = this.rm.getRMContext().getRMNodes().get(nid);
|
RMNode ni = this.rm.getRMContext().getRMNodes().get(nid);
|
||||||
boolean isInactive = false;
|
boolean isInactive = false;
|
||||||
if (ni == null) {
|
if (ni == null) {
|
||||||
|
@ -1467,9 +1468,7 @@ public class RMWebServices extends WebServices {
|
||||||
String error =
|
String error =
|
||||||
"Could not parse application id " + newApp.getApplicationId();
|
"Could not parse application id " + newApp.getApplicationId();
|
||||||
try {
|
try {
|
||||||
appid =
|
appid = ApplicationId.fromString(newApp.getApplicationId());
|
||||||
ConverterUtils.toApplicationId(recordFactory,
|
|
||||||
newApp.getApplicationId());
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new BadRequestException(error);
|
throw new BadRequestException(error);
|
||||||
}
|
}
|
||||||
|
@ -1553,7 +1552,7 @@ public class RMWebServices extends WebServices {
|
||||||
LocalResourceInfo l = entry.getValue();
|
LocalResourceInfo l = entry.getValue();
|
||||||
LocalResource lr =
|
LocalResource lr =
|
||||||
LocalResource.newInstance(
|
LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromURI(l.getUrl()), l.getType(),
|
URL.fromURI(l.getUrl()), l.getType(),
|
||||||
l.getVisibility(), l.getSize(), l.getTimestamp());
|
l.getVisibility(), l.getSize(), l.getTimestamp());
|
||||||
hlr.put(entry.getKey(), lr);
|
hlr.put(entry.getKey(), lr);
|
||||||
}
|
}
|
||||||
|
|
|
@ -67,7 +67,7 @@ public class AppAttemptInfo {
|
||||||
this.nodeId = masterContainer.getNodeId().toString();
|
this.nodeId = masterContainer.getNodeId().toString();
|
||||||
this.logsLink = WebAppUtils.getRunningLogURL(schemePrefix
|
this.logsLink = WebAppUtils.getRunningLogURL(schemePrefix
|
||||||
+ masterContainer.getNodeHttpAddress(),
|
+ masterContainer.getNodeHttpAddress(),
|
||||||
ConverterUtils.toString(masterContainer.getId()), user);
|
masterContainer.getId().toString(), user);
|
||||||
|
|
||||||
nodesBlacklistedBySystem =
|
nodesBlacklistedBySystem =
|
||||||
StringUtils.join(attempt.getAMBlacklistManager()
|
StringUtils.join(attempt.getAMBlacklistManager()
|
||||||
|
|
|
@ -165,8 +165,7 @@ public class AppInfo {
|
||||||
this.amContainerLogsExist = true;
|
this.amContainerLogsExist = true;
|
||||||
this.amContainerLogs = WebAppUtils.getRunningLogURL(
|
this.amContainerLogs = WebAppUtils.getRunningLogURL(
|
||||||
schemePrefix + masterContainer.getNodeHttpAddress(),
|
schemePrefix + masterContainer.getNodeHttpAddress(),
|
||||||
ConverterUtils.toString(masterContainer.getId()),
|
masterContainer.getId().toString(), app.getUser());
|
||||||
app.getUser());
|
|
||||||
this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
|
this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -218,7 +218,7 @@ public class TestRMAdminService {
|
||||||
fail("Should not get any exceptions");
|
fail("Should not get any exceptions");
|
||||||
}
|
}
|
||||||
|
|
||||||
NodeId nid = ConverterUtils.toNodeId("h1:1234");
|
NodeId nid = NodeId.fromString("h1:1234");
|
||||||
RMNode ni = rm.getRMContext().getRMNodes().get(nid);
|
RMNode ni = rm.getRMContext().getRMNodes().get(nid);
|
||||||
Resource resource = ni.getTotalCapability();
|
Resource resource = ni.getTotalCapability();
|
||||||
Assert.assertEquals("<memory:5120, vCores:5>", resource.toString());
|
Assert.assertEquals("<memory:5120, vCores:5>", resource.toString());
|
||||||
|
@ -257,7 +257,7 @@ public class TestRMAdminService {
|
||||||
fail("Should not get any exceptions");
|
fail("Should not get any exceptions");
|
||||||
}
|
}
|
||||||
|
|
||||||
NodeId nid = ConverterUtils.toNodeId("h1:1234");
|
NodeId nid = NodeId.fromString("h1:1234");
|
||||||
RMNode ni = rm.getRMContext().getRMNodes().get(nid);
|
RMNode ni = rm.getRMContext().getRMNodes().get(nid);
|
||||||
Resource resource = ni.getTotalCapability();
|
Resource resource = ni.getTotalCapability();
|
||||||
Assert.assertEquals("<memory:2048, vCores:2>", resource.toString());
|
Assert.assertEquals("<memory:2048, vCores:2>", resource.toString());
|
||||||
|
@ -307,7 +307,7 @@ public class TestRMAdminService {
|
||||||
fail("Should not get any exceptions");
|
fail("Should not get any exceptions");
|
||||||
}
|
}
|
||||||
|
|
||||||
NodeId nid = ConverterUtils.toNodeId("h1:1234");
|
NodeId nid = NodeId.fromString("h1:1234");
|
||||||
RMNode ni = rm.getRMContext().getRMNodes().get(nid);
|
RMNode ni = rm.getRMContext().getRMNodes().get(nid);
|
||||||
Resource resource = ni.getTotalCapability();
|
Resource resource = ni.getTotalCapability();
|
||||||
Assert.assertEquals("<memory:2048, vCores:2>", resource.toString());
|
Assert.assertEquals("<memory:2048, vCores:2>", resource.toString());
|
||||||
|
@ -355,7 +355,7 @@ public class TestRMAdminService {
|
||||||
fail("Should not get any exceptions");
|
fail("Should not get any exceptions");
|
||||||
}
|
}
|
||||||
|
|
||||||
NodeId nid = ConverterUtils.toNodeId("h1:1234");
|
NodeId nid = NodeId.fromString("h1:1234");
|
||||||
RMNode ni = rm.getRMContext().getRMNodes().get(nid);
|
RMNode ni = rm.getRMContext().getRMNodes().get(nid);
|
||||||
Resource resource = ni.getTotalCapability();
|
Resource resource = ni.getTotalCapability();
|
||||||
Assert.assertEquals("<memory:5120, vCores:5>", resource.toString());
|
Assert.assertEquals("<memory:5120, vCores:5>", resource.toString());
|
||||||
|
|
|
@ -183,7 +183,7 @@ public class RMStateStoreTestBase {
|
||||||
RMAppAttemptMetrics mockRmAppAttemptMetrics =
|
RMAppAttemptMetrics mockRmAppAttemptMetrics =
|
||||||
mock(RMAppAttemptMetrics.class);
|
mock(RMAppAttemptMetrics.class);
|
||||||
Container container = new ContainerPBImpl();
|
Container container = new ContainerPBImpl();
|
||||||
container.setId(ConverterUtils.toContainerId(containerIdStr));
|
container.setId(ContainerId.fromString(containerIdStr));
|
||||||
RMAppAttempt mockAttempt = mock(RMAppAttempt.class);
|
RMAppAttempt mockAttempt = mock(RMAppAttempt.class);
|
||||||
when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
|
when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
|
||||||
when(mockAttempt.getMasterContainer()).thenReturn(container);
|
when(mockAttempt.getMasterContainer()).thenReturn(container);
|
||||||
|
@ -227,8 +227,8 @@ public class RMStateStoreTestBase {
|
||||||
ClientToAMTokenSecretManagerInRM clientToAMTokenMgr =
|
ClientToAMTokenSecretManagerInRM clientToAMTokenMgr =
|
||||||
new ClientToAMTokenSecretManagerInRM();
|
new ClientToAMTokenSecretManagerInRM();
|
||||||
|
|
||||||
ApplicationAttemptId attemptId1 = ConverterUtils
|
ApplicationAttemptId attemptId1 = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId("appattempt_1352994193343_0001_000001");
|
"appattempt_1352994193343_0001_000001");
|
||||||
ApplicationId appId1 = attemptId1.getApplicationId();
|
ApplicationId appId1 = attemptId1.getApplicationId();
|
||||||
storeApp(store, appId1, submitTime, startTime);
|
storeApp(store, appId1, submitTime, startTime);
|
||||||
verifier.afterStoreApp(store, appId1);
|
verifier.afterStoreApp(store, appId1);
|
||||||
|
@ -245,8 +245,8 @@ public class RMStateStoreTestBase {
|
||||||
.getMasterContainer().getId();
|
.getMasterContainer().getId();
|
||||||
|
|
||||||
String appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
|
String appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
|
||||||
ApplicationAttemptId attemptId2 =
|
ApplicationAttemptId attemptId2 = ApplicationAttemptId.fromString(
|
||||||
ConverterUtils.toApplicationAttemptId(appAttemptIdStr2);
|
appAttemptIdStr2);
|
||||||
|
|
||||||
// create application token and client token key for attempt2
|
// create application token and client token key for attempt2
|
||||||
Token<AMRMTokenIdentifier> appAttemptToken2 =
|
Token<AMRMTokenIdentifier> appAttemptToken2 =
|
||||||
|
@ -259,8 +259,8 @@ public class RMStateStoreTestBase {
|
||||||
appAttemptToken2, clientTokenKey2, dispatcher)
|
appAttemptToken2, clientTokenKey2, dispatcher)
|
||||||
.getMasterContainer().getId();
|
.getMasterContainer().getId();
|
||||||
|
|
||||||
ApplicationAttemptId attemptIdRemoved = ConverterUtils
|
ApplicationAttemptId attemptIdRemoved = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId("appattempt_1352994193343_0002_000001");
|
"appattempt_1352994193343_0002_000001");
|
||||||
ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
|
ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
|
||||||
storeApp(store, appIdRemoved, submitTime, startTime);
|
storeApp(store, appIdRemoved, submitTime, startTime);
|
||||||
storeAttempt(store, attemptIdRemoved,
|
storeAttempt(store, attemptIdRemoved,
|
||||||
|
|
|
@ -183,7 +183,7 @@ public class TestFSRMStateStore extends RMStateStoreTestBase {
|
||||||
(FileSystemRMStateStore) fsTester.getRMStateStore();
|
(FileSystemRMStateStore) fsTester.getRMStateStore();
|
||||||
String appAttemptIdStr3 = "appattempt_1352994193343_0001_000003";
|
String appAttemptIdStr3 = "appattempt_1352994193343_0001_000003";
|
||||||
ApplicationAttemptId attemptId3 =
|
ApplicationAttemptId attemptId3 =
|
||||||
ConverterUtils.toApplicationAttemptId(appAttemptIdStr3);
|
ApplicationAttemptId.fromString(appAttemptIdStr3);
|
||||||
Path appDir =
|
Path appDir =
|
||||||
fsTester.store.getAppDir(attemptId3.getApplicationId().toString());
|
fsTester.store.getAppDir(attemptId3.getApplicationId().toString());
|
||||||
Path tempAppAttemptFile =
|
Path tempAppAttemptFile =
|
||||||
|
@ -364,7 +364,7 @@ public class TestFSRMStateStore extends RMStateStoreTestBase {
|
||||||
// imitate appAttemptFile1 is still .new, but old one is deleted
|
// imitate appAttemptFile1 is still .new, but old one is deleted
|
||||||
String appAttemptIdStr1 = "appattempt_1352994193343_0001_000001";
|
String appAttemptIdStr1 = "appattempt_1352994193343_0001_000001";
|
||||||
ApplicationAttemptId attemptId1 =
|
ApplicationAttemptId attemptId1 =
|
||||||
ConverterUtils.toApplicationAttemptId(appAttemptIdStr1);
|
ApplicationAttemptId.fromString(appAttemptIdStr1);
|
||||||
Path appDir =
|
Path appDir =
|
||||||
fsTester.store.getAppDir(attemptId1.getApplicationId().toString());
|
fsTester.store.getAppDir(attemptId1.getApplicationId().toString());
|
||||||
Path appAttemptFile1 =
|
Path appAttemptFile1 =
|
||||||
|
|
|
@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||||
import org.apache.hadoop.yarn.api.records.Container;
|
import org.apache.hadoop.yarn.api.records.Container;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
|
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
|
||||||
|
@ -399,14 +400,14 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
|
||||||
// Add a new attempt
|
// Add a new attempt
|
||||||
ClientToAMTokenSecretManagerInRM clientToAMTokenMgr =
|
ClientToAMTokenSecretManagerInRM clientToAMTokenMgr =
|
||||||
new ClientToAMTokenSecretManagerInRM();
|
new ClientToAMTokenSecretManagerInRM();
|
||||||
ApplicationAttemptId attemptId = ConverterUtils
|
ApplicationAttemptId attemptId = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId("appattempt_1234567894321_0001_000001");
|
"appattempt_1234567894321_0001_000001");
|
||||||
SecretKey clientTokenMasterKey =
|
SecretKey clientTokenMasterKey =
|
||||||
clientToAMTokenMgr.createMasterKey(attemptId);
|
clientToAMTokenMgr.createMasterKey(attemptId);
|
||||||
RMAppAttemptMetrics mockRmAppAttemptMetrics =
|
RMAppAttemptMetrics mockRmAppAttemptMetrics =
|
||||||
mock(RMAppAttemptMetrics.class);
|
mock(RMAppAttemptMetrics.class);
|
||||||
Container container = new ContainerPBImpl();
|
Container container = new ContainerPBImpl();
|
||||||
container.setId(ConverterUtils.toContainerId("container_1234567891234_0001_01_000001"));
|
container.setId(ContainerId.fromString("container_1234567891234_0001_01_000001"));
|
||||||
RMAppAttempt mockAttempt = mock(RMAppAttempt.class);
|
RMAppAttempt mockAttempt = mock(RMAppAttempt.class);
|
||||||
when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
|
when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
|
||||||
when(mockAttempt.getMasterContainer()).thenReturn(container);
|
when(mockAttempt.getMasterContainer()).thenReturn(container);
|
||||||
|
@ -491,8 +492,8 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
|
||||||
TestDispatcher dispatcher = new TestDispatcher();
|
TestDispatcher dispatcher = new TestDispatcher();
|
||||||
store.setRMDispatcher(dispatcher);
|
store.setRMDispatcher(dispatcher);
|
||||||
|
|
||||||
ApplicationAttemptId attemptIdRemoved = ConverterUtils
|
ApplicationAttemptId attemptIdRemoved = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId("appattempt_1352994193343_0002_000001");
|
"appattempt_1352994193343_0002_000001");
|
||||||
ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
|
ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
|
||||||
storeApp(store, appIdRemoved, submitTime, startTime);
|
storeApp(store, appIdRemoved, submitTime, startTime);
|
||||||
storeAttempt(store, attemptIdRemoved,
|
storeAttempt(store, attemptIdRemoved,
|
||||||
|
|
|
@ -57,6 +57,7 @@ import org.apache.hadoop.security.Credentials;
|
||||||
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
|
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
|
||||||
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
|
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
|
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
|
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResource;
|
import org.apache.hadoop.yarn.api.records.LocalResource;
|
||||||
|
@ -65,6 +66,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
||||||
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
|
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
|
||||||
import org.apache.hadoop.yarn.api.records.QueueACL;
|
import org.apache.hadoop.yarn.api.records.QueueACL;
|
||||||
import org.apache.hadoop.yarn.api.records.ReservationId;
|
import org.apache.hadoop.yarn.api.records.ReservationId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
|
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
|
||||||
|
@ -859,7 +861,7 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
|
||||||
|
|
||||||
RMApp app =
|
RMApp app =
|
||||||
rm.getRMContext().getRMApps()
|
rm.getRMContext().getRMApps()
|
||||||
.get(ConverterUtils.toApplicationId(appId));
|
.get(ApplicationId.fromString(appId));
|
||||||
assertEquals(appName, app.getName());
|
assertEquals(appName, app.getName());
|
||||||
assertEquals(webserviceUserName, app.getUser());
|
assertEquals(webserviceUserName, app.getUser());
|
||||||
assertEquals(2, app.getMaxAppAttempts());
|
assertEquals(2, app.getMaxAppAttempts());
|
||||||
|
@ -877,8 +879,7 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
|
||||||
Map<String, LocalResource> appLRs = ctx.getLocalResources();
|
Map<String, LocalResource> appLRs = ctx.getLocalResources();
|
||||||
assertTrue(appLRs.containsKey(lrKey));
|
assertTrue(appLRs.containsKey(lrKey));
|
||||||
LocalResource exampleLR = appLRs.get(lrKey);
|
LocalResource exampleLR = appLRs.get(lrKey);
|
||||||
assertEquals(ConverterUtils.getYarnUrlFromURI(y.getUrl()),
|
assertEquals(URL.fromURI(y.getUrl()), exampleLR.getResource());
|
||||||
exampleLR.getResource());
|
|
||||||
assertEquals(y.getSize(), exampleLR.getSize());
|
assertEquals(y.getSize(), exampleLR.getSize());
|
||||||
assertEquals(y.getTimestamp(), exampleLR.getTimestamp());
|
assertEquals(y.getTimestamp(), exampleLR.getTimestamp());
|
||||||
assertEquals(y.getType(), exampleLR.getType());
|
assertEquals(y.getType(), exampleLR.getType());
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue