YARN-1942. Deprecate toString/fromString methods from ConverterUtils and move them to records classes like ContainerId/ApplicationId, etc. (wangda)
This commit is contained in:
parent
3c9d31c55a
commit
d838c6443d
@ -58,6 +58,7 @@
|
|||||||
import org.apache.hadoop.yarn.api.ApplicationConstants;
|
import org.apache.hadoop.yarn.api.ApplicationConstants;
|
||||||
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
|
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -291,8 +292,7 @@ private static void configureTask(JobConf job, Task task,
|
|||||||
Credentials credentials, Token<JobTokenIdentifier> jt) throws IOException {
|
Credentials credentials, Token<JobTokenIdentifier> jt) throws IOException {
|
||||||
job.setCredentials(credentials);
|
job.setCredentials(credentials);
|
||||||
|
|
||||||
ApplicationAttemptId appAttemptId =
|
ApplicationAttemptId appAttemptId = ContainerId.fromString(
|
||||||
ConverterUtils.toContainerId(
|
|
||||||
System.getenv(Environment.CONTAINER_ID.name()))
|
System.getenv(Environment.CONTAINER_ID.name()))
|
||||||
.getApplicationAttemptId();
|
.getApplicationAttemptId();
|
||||||
LOG.debug("APPLICATION_ATTEMPT_ID: " + appAttemptId);
|
LOG.debug("APPLICATION_ATTEMPT_ID: " + appAttemptId);
|
||||||
|
@ -1544,7 +1544,7 @@ public static void main(String[] args) {
|
|||||||
validateInputParam(appSubmitTimeStr,
|
validateInputParam(appSubmitTimeStr,
|
||||||
ApplicationConstants.APP_SUBMIT_TIME_ENV);
|
ApplicationConstants.APP_SUBMIT_TIME_ENV);
|
||||||
|
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
ApplicationAttemptId applicationAttemptId =
|
ApplicationAttemptId applicationAttemptId =
|
||||||
containerId.getApplicationAttemptId();
|
containerId.getApplicationAttemptId();
|
||||||
if (applicationAttemptId != null) {
|
if (applicationAttemptId != null) {
|
||||||
|
@ -721,8 +721,7 @@ private static LocalResource createLocalResource(FileSystem fc, Path file,
|
|||||||
LocalResourceType type, LocalResourceVisibility visibility)
|
LocalResourceType type, LocalResourceVisibility visibility)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
FileStatus fstat = fc.getFileStatus(file);
|
FileStatus fstat = fc.getFileStatus(file);
|
||||||
URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
|
URL resourceURL = URL.fromPath(fc.resolvePath(fstat.getPath()));
|
||||||
.getPath()));
|
|
||||||
long resourceSize = fstat.getLen();
|
long resourceSize = fstat.getLen();
|
||||||
long resourceModificationTime = fstat.getModificationTime();
|
long resourceModificationTime = fstat.getModificationTime();
|
||||||
|
|
||||||
@ -1263,8 +1262,8 @@ public void setAvataar(Avataar avataar) {
|
|||||||
public TaskAttemptStateInternal recover(TaskAttemptInfo taInfo,
|
public TaskAttemptStateInternal recover(TaskAttemptInfo taInfo,
|
||||||
OutputCommitter committer, boolean recoverOutput) {
|
OutputCommitter committer, boolean recoverOutput) {
|
||||||
ContainerId containerId = taInfo.getContainerId();
|
ContainerId containerId = taInfo.getContainerId();
|
||||||
NodeId containerNodeId = ConverterUtils.toNodeId(taInfo.getHostname() + ":"
|
NodeId containerNodeId = NodeId.fromString(
|
||||||
+ taInfo.getPort());
|
taInfo.getHostname() + ":" + taInfo.getPort());
|
||||||
String nodeHttpAddress = StringInterner.weakIntern(taInfo.getHostname() + ":"
|
String nodeHttpAddress = StringInterner.weakIntern(taInfo.getHostname() + ":"
|
||||||
+ taInfo.getHttpPort());
|
+ taInfo.getHttpPort());
|
||||||
// Resource/Priority/Tokens are only needed while launching the container on
|
// Resource/Priority/Tokens are only needed while launching the container on
|
||||||
|
@ -69,8 +69,10 @@ public TaskAttemptInfo(TaskAttempt ta, TaskType type, Boolean isRunning) {
|
|||||||
this.nodeHttpAddress = ta.getNodeHttpAddress();
|
this.nodeHttpAddress = ta.getNodeHttpAddress();
|
||||||
this.startTime = report.getStartTime();
|
this.startTime = report.getStartTime();
|
||||||
this.finishTime = report.getFinishTime();
|
this.finishTime = report.getFinishTime();
|
||||||
this.assignedContainerId = ConverterUtils.toString(report.getContainerId());
|
|
||||||
this.assignedContainer = report.getContainerId();
|
this.assignedContainer = report.getContainerId();
|
||||||
|
if (assignedContainer != null) {
|
||||||
|
this.assignedContainerId = assignedContainer.toString();
|
||||||
|
}
|
||||||
this.progress = report.getProgress() * 100;
|
this.progress = report.getProgress() * 100;
|
||||||
this.status = report.getStateString();
|
this.status = report.getStateString();
|
||||||
this.state = report.getTaskAttemptState();
|
this.state = report.getTaskAttemptState();
|
||||||
|
@ -136,9 +136,9 @@ public void testMRAppMasterForDifferentUser() throws IOException,
|
|||||||
String containerIdStr = "container_1317529182569_0004_000001_1";
|
String containerIdStr = "container_1317529182569_0004_000001_1";
|
||||||
|
|
||||||
String userName = "TestAppMasterUser";
|
String userName = "TestAppMasterUser";
|
||||||
ApplicationAttemptId applicationAttemptId = ConverterUtils
|
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId(applicationAttemptIdStr);
|
applicationAttemptIdStr);
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
MRAppMasterTest appMaster =
|
MRAppMasterTest appMaster =
|
||||||
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
||||||
System.currentTimeMillis());
|
System.currentTimeMillis());
|
||||||
@ -159,15 +159,15 @@ public void testMRAppMasterMidLock() throws IOException,
|
|||||||
String userName = "TestAppMasterUser";
|
String userName = "TestAppMasterUser";
|
||||||
JobConf conf = new JobConf();
|
JobConf conf = new JobConf();
|
||||||
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
||||||
ApplicationAttemptId applicationAttemptId = ConverterUtils
|
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId(applicationAttemptIdStr);
|
applicationAttemptIdStr);
|
||||||
JobId jobId = TypeConverter.toYarn(
|
JobId jobId = TypeConverter.toYarn(
|
||||||
TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
|
TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
|
||||||
Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
|
Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
|
||||||
FileSystem fs = FileSystem.get(conf);
|
FileSystem fs = FileSystem.get(conf);
|
||||||
//Create the file, but no end file so we should unregister with an error.
|
//Create the file, but no end file so we should unregister with an error.
|
||||||
fs.create(start).close();
|
fs.create(start).close();
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
MRAppMaster appMaster =
|
MRAppMaster appMaster =
|
||||||
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
||||||
System.currentTimeMillis(), false, false);
|
System.currentTimeMillis(), false, false);
|
||||||
@ -198,8 +198,8 @@ public void testMRAppMasterJobLaunchTime() throws IOException,
|
|||||||
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
||||||
conf.setInt(MRJobConfig.NUM_REDUCES, 0);
|
conf.setInt(MRJobConfig.NUM_REDUCES, 0);
|
||||||
conf.set(JHAdminConfig.MR_HS_JHIST_FORMAT, "json");
|
conf.set(JHAdminConfig.MR_HS_JHIST_FORMAT, "json");
|
||||||
ApplicationAttemptId applicationAttemptId = ConverterUtils
|
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId(applicationAttemptIdStr);
|
applicationAttemptIdStr);
|
||||||
JobId jobId = TypeConverter.toYarn(
|
JobId jobId = TypeConverter.toYarn(
|
||||||
TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
|
TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
|
||||||
|
|
||||||
@ -217,7 +217,7 @@ public void testMRAppMasterJobLaunchTime() throws IOException,
|
|||||||
FileSystem fs = FileSystem.get(conf);
|
FileSystem fs = FileSystem.get(conf);
|
||||||
JobSplitWriter.createSplitFiles(new Path(dir.getAbsolutePath()), conf,
|
JobSplitWriter.createSplitFiles(new Path(dir.getAbsolutePath()), conf,
|
||||||
fs, new org.apache.hadoop.mapred.InputSplit[0]);
|
fs, new org.apache.hadoop.mapred.InputSplit[0]);
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
MRAppMasterTestLaunchTime appMaster =
|
MRAppMasterTestLaunchTime appMaster =
|
||||||
new MRAppMasterTestLaunchTime(applicationAttemptId, containerId,
|
new MRAppMasterTestLaunchTime(applicationAttemptId, containerId,
|
||||||
"host", -1, -1, System.currentTimeMillis());
|
"host", -1, -1, System.currentTimeMillis());
|
||||||
@ -235,8 +235,8 @@ public void testMRAppMasterSuccessLock() throws IOException,
|
|||||||
String userName = "TestAppMasterUser";
|
String userName = "TestAppMasterUser";
|
||||||
JobConf conf = new JobConf();
|
JobConf conf = new JobConf();
|
||||||
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
||||||
ApplicationAttemptId applicationAttemptId = ConverterUtils
|
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId(applicationAttemptIdStr);
|
applicationAttemptIdStr);
|
||||||
JobId jobId = TypeConverter.toYarn(
|
JobId jobId = TypeConverter.toYarn(
|
||||||
TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
|
TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
|
||||||
Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
|
Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
|
||||||
@ -244,7 +244,7 @@ public void testMRAppMasterSuccessLock() throws IOException,
|
|||||||
FileSystem fs = FileSystem.get(conf);
|
FileSystem fs = FileSystem.get(conf);
|
||||||
fs.create(start).close();
|
fs.create(start).close();
|
||||||
fs.create(end).close();
|
fs.create(end).close();
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
MRAppMaster appMaster =
|
MRAppMaster appMaster =
|
||||||
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
||||||
System.currentTimeMillis(), false, false);
|
System.currentTimeMillis(), false, false);
|
||||||
@ -273,8 +273,8 @@ public void testMRAppMasterFailLock() throws IOException,
|
|||||||
String userName = "TestAppMasterUser";
|
String userName = "TestAppMasterUser";
|
||||||
JobConf conf = new JobConf();
|
JobConf conf = new JobConf();
|
||||||
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
||||||
ApplicationAttemptId applicationAttemptId = ConverterUtils
|
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId(applicationAttemptIdStr);
|
applicationAttemptIdStr);
|
||||||
JobId jobId = TypeConverter.toYarn(
|
JobId jobId = TypeConverter.toYarn(
|
||||||
TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
|
TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
|
||||||
Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
|
Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
|
||||||
@ -282,7 +282,7 @@ public void testMRAppMasterFailLock() throws IOException,
|
|||||||
FileSystem fs = FileSystem.get(conf);
|
FileSystem fs = FileSystem.get(conf);
|
||||||
fs.create(start).close();
|
fs.create(start).close();
|
||||||
fs.create(end).close();
|
fs.create(end).close();
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
MRAppMaster appMaster =
|
MRAppMaster appMaster =
|
||||||
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
||||||
System.currentTimeMillis(), false, false);
|
System.currentTimeMillis(), false, false);
|
||||||
@ -311,8 +311,8 @@ public void testMRAppMasterMissingStaging() throws IOException,
|
|||||||
String userName = "TestAppMasterUser";
|
String userName = "TestAppMasterUser";
|
||||||
JobConf conf = new JobConf();
|
JobConf conf = new JobConf();
|
||||||
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
||||||
ApplicationAttemptId applicationAttemptId = ConverterUtils
|
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId(applicationAttemptIdStr);
|
applicationAttemptIdStr);
|
||||||
|
|
||||||
//Delete the staging directory
|
//Delete the staging directory
|
||||||
File dir = new File(stagingDir);
|
File dir = new File(stagingDir);
|
||||||
@ -320,7 +320,7 @@ public void testMRAppMasterMissingStaging() throws IOException,
|
|||||||
FileUtils.deleteDirectory(dir);
|
FileUtils.deleteDirectory(dir);
|
||||||
}
|
}
|
||||||
|
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
MRAppMaster appMaster =
|
MRAppMaster appMaster =
|
||||||
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
|
||||||
System.currentTimeMillis(), false, false);
|
System.currentTimeMillis(), false, false);
|
||||||
@ -351,9 +351,9 @@ public void testMRAppMasterMaxAppAttempts() throws IOException,
|
|||||||
String containerIdStr = "container_1317529182569_0004_000002_1";
|
String containerIdStr = "container_1317529182569_0004_000002_1";
|
||||||
|
|
||||||
String userName = "TestAppMasterUser";
|
String userName = "TestAppMasterUser";
|
||||||
ApplicationAttemptId applicationAttemptId = ConverterUtils
|
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId(applicationAttemptIdStr);
|
applicationAttemptIdStr);
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
JobConf conf = new JobConf();
|
JobConf conf = new JobConf();
|
||||||
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
||||||
|
|
||||||
@ -505,9 +505,9 @@ public void testMRAppMasterShutDownJob() throws Exception,
|
|||||||
String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
|
String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
|
||||||
String containerIdStr = "container_1317529182569_0004_000002_1";
|
String containerIdStr = "container_1317529182569_0004_000002_1";
|
||||||
String userName = "TestAppMasterUser";
|
String userName = "TestAppMasterUser";
|
||||||
ApplicationAttemptId applicationAttemptId = ConverterUtils
|
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId(applicationAttemptIdStr);
|
applicationAttemptIdStr);
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
JobConf conf = new JobConf();
|
JobConf conf = new JobConf();
|
||||||
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
||||||
|
|
||||||
|
@ -129,8 +129,8 @@ public void testCommitWindow() throws Exception {
|
|||||||
|
|
||||||
SystemClock clock = new SystemClock();
|
SystemClock clock = new SystemClock();
|
||||||
AppContext appContext = mock(AppContext.class);
|
AppContext appContext = mock(AppContext.class);
|
||||||
ApplicationAttemptId attemptid =
|
ApplicationAttemptId attemptid = ApplicationAttemptId.fromString(
|
||||||
ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
|
"appattempt_1234567890000_0001_0");
|
||||||
when(appContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
|
when(appContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
|
||||||
when(appContext.getApplicationAttemptId()).thenReturn(attemptid);
|
when(appContext.getApplicationAttemptId()).thenReturn(attemptid);
|
||||||
when(appContext.getEventHandler()).thenReturn(
|
when(appContext.getEventHandler()).thenReturn(
|
||||||
@ -240,8 +240,8 @@ public void testBasic() throws Exception {
|
|||||||
YarnConfiguration conf = new YarnConfiguration();
|
YarnConfiguration conf = new YarnConfiguration();
|
||||||
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
||||||
JobContext mockJobContext = mock(JobContext.class);
|
JobContext mockJobContext = mock(JobContext.class);
|
||||||
ApplicationAttemptId attemptid =
|
ApplicationAttemptId attemptid = ApplicationAttemptId.fromString(
|
||||||
ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
|
"appattempt_1234567890000_0001_0");
|
||||||
JobId jobId = TypeConverter.toYarn(
|
JobId jobId = TypeConverter.toYarn(
|
||||||
TypeConverter.fromYarn(attemptid.getApplicationId()));
|
TypeConverter.fromYarn(attemptid.getApplicationId()));
|
||||||
|
|
||||||
@ -289,7 +289,7 @@ public void testFailure() throws Exception {
|
|||||||
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
|
||||||
JobContext mockJobContext = mock(JobContext.class);
|
JobContext mockJobContext = mock(JobContext.class);
|
||||||
ApplicationAttemptId attemptid =
|
ApplicationAttemptId attemptid =
|
||||||
ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
|
ApplicationAttemptId.fromString("appattempt_1234567890000_0001_0");
|
||||||
JobId jobId = TypeConverter.toYarn(
|
JobId jobId = TypeConverter.toYarn(
|
||||||
TypeConverter.fromYarn(attemptid.getApplicationId()));
|
TypeConverter.fromYarn(attemptid.getApplicationId()));
|
||||||
|
|
||||||
|
@ -941,8 +941,8 @@ public void runOnNextHeartbeat(Runnable callback) {
|
|||||||
callback.run();
|
callback.run();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
ApplicationAttemptId id =
|
ApplicationAttemptId id = ApplicationAttemptId.fromString(
|
||||||
ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
|
"appattempt_1234567890000_0001_0");
|
||||||
when(appContext.getApplicationID()).thenReturn(id.getApplicationId());
|
when(appContext.getApplicationID()).thenReturn(id.getApplicationId());
|
||||||
when(appContext.getApplicationAttemptId()).thenReturn(id);
|
when(appContext.getApplicationAttemptId()).thenReturn(id);
|
||||||
CommitterEventHandler handler =
|
CommitterEventHandler handler =
|
||||||
|
@ -515,7 +515,7 @@ public void verifyTaskAttemptGeneric(TaskAttempt ta, TaskType ttype,
|
|||||||
WebServicesTestUtils.checkStringMatch("diagnostics", expectDiag,
|
WebServicesTestUtils.checkStringMatch("diagnostics", expectDiag,
|
||||||
diagnostics);
|
diagnostics);
|
||||||
WebServicesTestUtils.checkStringMatch("assignedContainerId",
|
WebServicesTestUtils.checkStringMatch("assignedContainerId",
|
||||||
ConverterUtils.toString(ta.getAssignedContainerID()),
|
ta.getAssignedContainerID().toString(),
|
||||||
assignedContainerId);
|
assignedContainerId);
|
||||||
|
|
||||||
assertEquals("startTime wrong", ta.getLaunchTime(), startTime);
|
assertEquals("startTime wrong", ta.getLaunchTime(), startTime);
|
||||||
|
@ -158,7 +158,7 @@ public void setup(JobConf conf) throws IOException {
|
|||||||
}
|
}
|
||||||
Path resourcePath;
|
Path resourcePath;
|
||||||
try {
|
try {
|
||||||
resourcePath = ConverterUtils.getPathFromYarnURL(resource.getResource());
|
resourcePath = resource.getResource().toPath();
|
||||||
} catch (URISyntaxException e) {
|
} catch (URISyntaxException e) {
|
||||||
throw new IOException(e);
|
throw new IOException(e);
|
||||||
}
|
}
|
||||||
|
@ -68,6 +68,7 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.LocalResource;
|
import org.apache.hadoop.yarn.api.records.LocalResource;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResourceType;
|
import org.apache.hadoop.yarn.api.records.LocalResourceType;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
||||||
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
||||||
import org.apache.hadoop.yarn.util.Apps;
|
import org.apache.hadoop.yarn.util.Apps;
|
||||||
@ -608,8 +609,7 @@ private static void parseDistributedCacheArtifacts(
|
|||||||
}
|
}
|
||||||
String linkName = name.toUri().getPath();
|
String linkName = name.toUri().getPath();
|
||||||
LocalResource orig = localResources.get(linkName);
|
LocalResource orig = localResources.get(linkName);
|
||||||
org.apache.hadoop.yarn.api.records.URL url =
|
URL url = URL.fromURI(p.toUri());
|
||||||
ConverterUtils.getYarnUrlFromURI(p.toUri());
|
|
||||||
if(orig != null && !orig.getResource().equals(url)) {
|
if(orig != null && !orig.getResource().equals(url)) {
|
||||||
LOG.warn(
|
LOG.warn(
|
||||||
getResourceDescription(orig.getType()) +
|
getResourceDescription(orig.getType()) +
|
||||||
@ -618,8 +618,8 @@ private static void parseDistributedCacheArtifacts(
|
|||||||
" This will be an error in Hadoop 2.0");
|
" This will be an error in Hadoop 2.0");
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
localResources.put(linkName, LocalResource.newInstance(ConverterUtils
|
localResources.put(linkName, LocalResource
|
||||||
.getYarnUrlFromURI(p.toUri()), type, visibilities[i]
|
.newInstance(URL.fromURI(p.toUri()), type, visibilities[i]
|
||||||
? LocalResourceVisibility.PUBLIC : LocalResourceVisibility.PRIVATE,
|
? LocalResourceVisibility.PUBLIC : LocalResourceVisibility.PRIVATE,
|
||||||
sizes[i], timestamps[i]));
|
sizes[i], timestamps[i]));
|
||||||
}
|
}
|
||||||
|
@ -107,8 +107,8 @@ public void setDatum(Object datum) {
|
|||||||
* @return the ApplicationAttemptId
|
* @return the ApplicationAttemptId
|
||||||
*/
|
*/
|
||||||
public ApplicationAttemptId getAppAttemptId() {
|
public ApplicationAttemptId getAppAttemptId() {
|
||||||
return ConverterUtils.toApplicationAttemptId(datum.getApplicationAttemptId()
|
return ApplicationAttemptId.fromString(
|
||||||
.toString());
|
datum.getApplicationAttemptId().toString());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -122,7 +122,7 @@ public long getStartTime() {
|
|||||||
* @return the ContainerId for the MRAppMaster.
|
* @return the ContainerId for the MRAppMaster.
|
||||||
*/
|
*/
|
||||||
public ContainerId getContainerId() {
|
public ContainerId getContainerId() {
|
||||||
return ConverterUtils.toContainerId(datum.getContainerId().toString());
|
return ContainerId.fromString(datum.getContainerId().toString());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -75,7 +75,7 @@ public TaskAttemptStartedEvent(TaskAttemptID attemptId, TaskType taskType,
|
|||||||
long startTime, String trackerName, int httpPort, int shufflePort,
|
long startTime, String trackerName, int httpPort, int shufflePort,
|
||||||
String locality, String avataar) {
|
String locality, String avataar) {
|
||||||
this(attemptId, taskType, startTime, trackerName, httpPort, shufflePort,
|
this(attemptId, taskType, startTime, trackerName, httpPort, shufflePort,
|
||||||
ConverterUtils.toContainerId("container_-1_-1_-1_-1"), locality,
|
ContainerId.fromString("container_-1_-1_-1_-1"), locality,
|
||||||
avataar);
|
avataar);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -116,7 +116,7 @@ public EventType getEventType() {
|
|||||||
}
|
}
|
||||||
/** Get the ContainerId */
|
/** Get the ContainerId */
|
||||||
public ContainerId getContainerId() {
|
public ContainerId getContainerId() {
|
||||||
return ConverterUtils.toContainerId(datum.getContainerId().toString());
|
return ContainerId.fromString(datum.getContainerId().toString());
|
||||||
}
|
}
|
||||||
/** Get the locality */
|
/** Get the locality */
|
||||||
public String getLocality() {
|
public String getLocality() {
|
||||||
|
@ -533,7 +533,7 @@ public void verifyTaskAttemptGeneric(TaskAttempt ta, TaskType ttype,
|
|||||||
WebServicesTestUtils.checkStringMatch("diagnostics", expectDiag,
|
WebServicesTestUtils.checkStringMatch("diagnostics", expectDiag,
|
||||||
diagnostics);
|
diagnostics);
|
||||||
WebServicesTestUtils.checkStringMatch("assignedContainerId",
|
WebServicesTestUtils.checkStringMatch("assignedContainerId",
|
||||||
ConverterUtils.toString(ta.getAssignedContainerID()),
|
ta.getAssignedContainerID().toString(),
|
||||||
assignedContainerId);
|
assignedContainerId);
|
||||||
|
|
||||||
assertEquals("startTime wrong", ta.getLaunchTime(), startTime);
|
assertEquals("startTime wrong", ta.getLaunchTime(), startTime);
|
||||||
|
@ -321,7 +321,7 @@ private LocalResource createApplicationResource(FileContext fs, Path p, LocalRes
|
|||||||
throws IOException {
|
throws IOException {
|
||||||
LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
FileStatus rsrcStat = fs.getFileStatus(p);
|
FileStatus rsrcStat = fs.getFileStatus(p);
|
||||||
rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs
|
rsrc.setResource(URL.fromPath(fs
|
||||||
.getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
|
.getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
|
||||||
rsrc.setSize(rsrcStat.getLen());
|
rsrc.setSize(rsrcStat.getLen());
|
||||||
rsrc.setTimestamp(rsrcStat.getModificationTime());
|
rsrc.setTimestamp(rsrcStat.getModificationTime());
|
||||||
@ -355,8 +355,7 @@ public ApplicationSubmissionContext createApplicationSubmissionContext(
|
|||||||
|
|
||||||
Path jobConfPath = new Path(jobSubmitDir, MRJobConfig.JOB_CONF_FILE);
|
Path jobConfPath = new Path(jobSubmitDir, MRJobConfig.JOB_CONF_FILE);
|
||||||
|
|
||||||
URL yarnUrlForJobSubmitDir = ConverterUtils
|
URL yarnUrlForJobSubmitDir = URL.fromPath(defaultFileContext.getDefaultFileSystem()
|
||||||
.getYarnUrlFromPath(defaultFileContext.getDefaultFileSystem()
|
|
||||||
.resolvePath(
|
.resolvePath(
|
||||||
defaultFileContext.makeQualified(new Path(jobSubmitDir))));
|
defaultFileContext.makeQualified(new Path(jobSubmitDir))));
|
||||||
LOG.debug("Creating setup context, jobSubmitDir url is "
|
LOG.debug("Creating setup context, jobSubmitDir url is "
|
||||||
|
@ -758,7 +758,7 @@ public void testContainerRollingLog() throws IOException,
|
|||||||
boolean foundAppMaster = job.isUber();
|
boolean foundAppMaster = job.isUber();
|
||||||
final Path containerPathComponent = slog.getPath().getParent();
|
final Path containerPathComponent = slog.getPath().getParent();
|
||||||
if (!foundAppMaster) {
|
if (!foundAppMaster) {
|
||||||
final ContainerId cid = ConverterUtils.toContainerId(
|
final ContainerId cid = ContainerId.fromString(
|
||||||
containerPathComponent.getName());
|
containerPathComponent.getName());
|
||||||
foundAppMaster =
|
foundAppMaster =
|
||||||
((cid.getContainerId() & ContainerId.CONTAINER_ID_BITMASK)== 1);
|
((cid.getContainerId() & ContainerId.CONTAINER_ID_BITMASK)== 1);
|
||||||
|
@ -999,7 +999,7 @@ private String getBaseLocation(String jobId, String user) {
|
|||||||
final String baseStr =
|
final String baseStr =
|
||||||
ContainerLocalizer.USERCACHE + "/" + user + "/"
|
ContainerLocalizer.USERCACHE + "/" + user + "/"
|
||||||
+ ContainerLocalizer.APPCACHE + "/"
|
+ ContainerLocalizer.APPCACHE + "/"
|
||||||
+ ConverterUtils.toString(appID) + "/output" + "/";
|
+ appID.toString() + "/output" + "/";
|
||||||
return baseStr;
|
return baseStr;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -39,6 +39,7 @@
|
|||||||
import org.apache.hadoop.mapred.JobConf;
|
import org.apache.hadoop.mapred.JobConf;
|
||||||
import org.apache.hadoop.util.Tool;
|
import org.apache.hadoop.util.Tool;
|
||||||
import org.apache.hadoop.util.ToolRunner;
|
import org.apache.hadoop.util.ToolRunner;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
||||||
import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
|
import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
|
||||||
import org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster;
|
import org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster;
|
||||||
@ -302,7 +303,7 @@ void filterAppsByAggregatedStatus() throws IOException, YarnException {
|
|||||||
AppInfo app = it.next();
|
AppInfo app = it.next();
|
||||||
try {
|
try {
|
||||||
ApplicationReport report = client.getApplicationReport(
|
ApplicationReport report = client.getApplicationReport(
|
||||||
ConverterUtils.toApplicationId(app.getAppId()));
|
ApplicationId.fromString(app.getAppId()));
|
||||||
LogAggregationStatus aggStatus = report.getLogAggregationStatus();
|
LogAggregationStatus aggStatus = report.getLogAggregationStatus();
|
||||||
if (aggStatus.equals(LogAggregationStatus.RUNNING) ||
|
if (aggStatus.equals(LogAggregationStatus.RUNNING) ||
|
||||||
aggStatus.equals(LogAggregationStatus.RUNNING_WITH_FAILURE) ||
|
aggStatus.equals(LogAggregationStatus.RUNNING_WITH_FAILURE) ||
|
||||||
|
@ -19,6 +19,8 @@
|
|||||||
package org.apache.hadoop.yarn.api.records;
|
package org.apache.hadoop.yarn.api.records;
|
||||||
|
|
||||||
import java.text.NumberFormat;
|
import java.text.NumberFormat;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.NoSuchElementException;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
@ -26,6 +28,8 @@
|
|||||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
|
|
||||||
|
import com.google.common.base.Splitter;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* <p><code>ApplicationAttemptId</code> denotes the particular <em>attempt</em>
|
* <p><code>ApplicationAttemptId</code> denotes the particular <em>attempt</em>
|
||||||
* of an <code>ApplicationMaster</code> for a given {@link ApplicationId}.</p>
|
* of an <code>ApplicationMaster</code> for a given {@link ApplicationId}.</p>
|
||||||
@ -38,10 +42,11 @@
|
|||||||
@Stable
|
@Stable
|
||||||
public abstract class ApplicationAttemptId implements
|
public abstract class ApplicationAttemptId implements
|
||||||
Comparable<ApplicationAttemptId> {
|
Comparable<ApplicationAttemptId> {
|
||||||
|
private static Splitter _spliter = Splitter.on('_').trimResults();
|
||||||
|
|
||||||
@Private
|
@Private
|
||||||
@Unstable
|
@Unstable
|
||||||
public static final String appAttemptIdStrPrefix = "appattempt_";
|
public static final String appAttemptIdStrPrefix = "appattempt";
|
||||||
|
|
||||||
@Public
|
@Public
|
||||||
@Unstable
|
@Unstable
|
||||||
@ -131,6 +136,7 @@ public int compareTo(ApplicationAttemptId other) {
|
|||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
StringBuilder sb = new StringBuilder(appAttemptIdStrPrefix);
|
StringBuilder sb = new StringBuilder(appAttemptIdStrPrefix);
|
||||||
|
sb.append("_");
|
||||||
sb.append(this.getApplicationId().getClusterTimestamp()).append("_");
|
sb.append(this.getApplicationId().getClusterTimestamp()).append("_");
|
||||||
sb.append(ApplicationId.appIdFormat.get().format(
|
sb.append(ApplicationId.appIdFormat.get().format(
|
||||||
this.getApplicationId().getId()));
|
this.getApplicationId().getId()));
|
||||||
@ -139,4 +145,32 @@ public String toString() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
protected abstract void build();
|
protected abstract void build();
|
||||||
|
|
||||||
|
@Public
|
||||||
|
@Stable
|
||||||
|
public static ApplicationAttemptId fromString(String applicationAttemptIdStr) {
|
||||||
|
Iterator<String> it = _spliter.split(applicationAttemptIdStr).iterator();
|
||||||
|
if (!it.next().equals(appAttemptIdStrPrefix)) {
|
||||||
|
throw new IllegalArgumentException("Invalid AppAttemptId prefix: "
|
||||||
|
+ applicationAttemptIdStr);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return toApplicationAttemptId(it);
|
||||||
|
} catch (NumberFormatException n) {
|
||||||
|
throw new IllegalArgumentException("Invalid AppAttemptId: "
|
||||||
|
+ applicationAttemptIdStr, n);
|
||||||
|
} catch (NoSuchElementException e) {
|
||||||
|
throw new IllegalArgumentException("Invalid AppAttemptId: "
|
||||||
|
+ applicationAttemptIdStr, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ApplicationAttemptId toApplicationAttemptId(
|
||||||
|
Iterator<String> it) throws NumberFormatException {
|
||||||
|
ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
|
||||||
|
Integer.parseInt(it.next()));
|
||||||
|
ApplicationAttemptId appAttemptId =
|
||||||
|
ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next()));
|
||||||
|
return appAttemptId;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -19,6 +19,8 @@
|
|||||||
package org.apache.hadoop.yarn.api.records;
|
package org.apache.hadoop.yarn.api.records;
|
||||||
|
|
||||||
import java.text.NumberFormat;
|
import java.text.NumberFormat;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.NoSuchElementException;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
@ -26,6 +28,8 @@
|
|||||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
|
|
||||||
|
import com.google.common.base.Splitter;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* <p><code>ApplicationId</code> represents the <em>globally unique</em>
|
* <p><code>ApplicationId</code> represents the <em>globally unique</em>
|
||||||
* identifier for an application.</p>
|
* identifier for an application.</p>
|
||||||
@ -38,10 +42,11 @@
|
|||||||
@Public
|
@Public
|
||||||
@Stable
|
@Stable
|
||||||
public abstract class ApplicationId implements Comparable<ApplicationId> {
|
public abstract class ApplicationId implements Comparable<ApplicationId> {
|
||||||
|
private static Splitter _spliter = Splitter.on('_').trimResults();
|
||||||
|
|
||||||
@Private
|
@Private
|
||||||
@Unstable
|
@Unstable
|
||||||
public static final String appIdStrPrefix = "application_";
|
public static final String appIdStrPrefix = "application";
|
||||||
|
|
||||||
@Public
|
@Public
|
||||||
@Unstable
|
@Unstable
|
||||||
@ -105,8 +110,35 @@ public int compareTo(ApplicationId other) {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return appIdStrPrefix + this.getClusterTimestamp() + "_"
|
return appIdStrPrefix + "_" + this.getClusterTimestamp() + "_" + appIdFormat
|
||||||
+ appIdFormat.get().format(getId());
|
.get().format(getId());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ApplicationId toApplicationId(
|
||||||
|
Iterator<String> it) throws NumberFormatException {
|
||||||
|
ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
|
||||||
|
Integer.parseInt(it.next()));
|
||||||
|
return appId;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Public
|
||||||
|
@Stable
|
||||||
|
public static ApplicationId fromString(String appIdStr) {
|
||||||
|
Iterator<String> it = _spliter.split((appIdStr)).iterator();
|
||||||
|
if (!it.next().equals(appIdStrPrefix)) {
|
||||||
|
throw new IllegalArgumentException("Invalid ApplicationId prefix: "
|
||||||
|
+ appIdStr + ". The valid ApplicationId should start with prefix "
|
||||||
|
+ appIdStrPrefix);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return toApplicationId(it);
|
||||||
|
} catch (NumberFormatException n) {
|
||||||
|
throw new IllegalArgumentException("Invalid ApplicationId: "
|
||||||
|
+ appIdStr, n);
|
||||||
|
} catch (NoSuchElementException e) {
|
||||||
|
throw new IllegalArgumentException("Invalid ApplicationId: "
|
||||||
|
+ appIdStr, e);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -42,7 +42,7 @@ public abstract class ContainerId implements Comparable<ContainerId>{
|
|||||||
private static final String CONTAINER_PREFIX = "container";
|
private static final String CONTAINER_PREFIX = "container";
|
||||||
private static final String EPOCH_PREFIX = "e";
|
private static final String EPOCH_PREFIX = "e";
|
||||||
|
|
||||||
@Private
|
@Public
|
||||||
@Unstable
|
@Unstable
|
||||||
public static ContainerId newContainerId(ApplicationAttemptId appAttemptId,
|
public static ContainerId newContainerId(ApplicationAttemptId appAttemptId,
|
||||||
long containerId) {
|
long containerId) {
|
||||||
@ -97,7 +97,7 @@ public static ContainerId newInstance(ApplicationAttemptId appAttemptId,
|
|||||||
*/
|
*/
|
||||||
@Public
|
@Public
|
||||||
@Deprecated
|
@Deprecated
|
||||||
@Stable
|
@Unstable
|
||||||
public abstract int getId();
|
public abstract int getId();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -205,7 +205,7 @@ public String toString() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Public
|
@Public
|
||||||
@Unstable
|
@Stable
|
||||||
public static ContainerId fromString(String containerIdStr) {
|
public static ContainerId fromString(String containerIdStr) {
|
||||||
Iterator<String> it = _SPLITTER.split(containerIdStr).iterator();
|
Iterator<String> it = _SPLITTER.split(containerIdStr).iterator();
|
||||||
if (!it.next().equals(CONTAINER_PREFIX)) {
|
if (!it.next().equals(CONTAINER_PREFIX)) {
|
||||||
|
@ -20,8 +20,8 @@
|
|||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
import org.apache.hadoop.classification.InterfaceStability.Stable;
|
|
||||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability.Stable;
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -35,8 +35,8 @@
|
|||||||
@Stable
|
@Stable
|
||||||
public abstract class NodeId implements Comparable<NodeId> {
|
public abstract class NodeId implements Comparable<NodeId> {
|
||||||
|
|
||||||
@Private
|
@Public
|
||||||
@Unstable
|
@Stable
|
||||||
public static NodeId newInstance(String host, int port) {
|
public static NodeId newInstance(String host, int port) {
|
||||||
NodeId nodeId = Records.newRecord(NodeId.class);
|
NodeId nodeId = Records.newRecord(NodeId.class);
|
||||||
nodeId.setHost(host);
|
nodeId.setHost(host);
|
||||||
@ -113,5 +113,22 @@ public int compareTo(NodeId other) {
|
|||||||
return hostCompare;
|
return hostCompare;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Public
|
||||||
|
@Stable
|
||||||
|
public static NodeId fromString(String nodeIdStr) {
|
||||||
|
String[] parts = nodeIdStr.split(":");
|
||||||
|
if (parts.length != 2) {
|
||||||
|
throw new IllegalArgumentException("Invalid NodeId [" + nodeIdStr
|
||||||
|
+ "]. Expected host:port");
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
NodeId nodeId =
|
||||||
|
NodeId.newInstance(parts[0].trim(), Integer.parseInt(parts[1]));
|
||||||
|
return nodeId;
|
||||||
|
} catch (NumberFormatException e) {
|
||||||
|
throw new IllegalArgumentException("Invalid port: " + parts[1], e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
protected abstract void build();
|
protected abstract void build();
|
||||||
}
|
}
|
||||||
|
@ -18,8 +18,13 @@
|
|||||||
|
|
||||||
package org.apache.hadoop.yarn.api.records;
|
package org.apache.hadoop.yarn.api.records;
|
||||||
|
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
import org.apache.hadoop.classification.InterfaceStability.Stable;
|
import org.apache.hadoop.classification.InterfaceStability.Stable;
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -119,4 +124,48 @@ public static URL newInstance(String scheme, String host, int port, String file)
|
|||||||
@Public
|
@Public
|
||||||
@Stable
|
@Stable
|
||||||
public abstract void setFile(String file);
|
public abstract void setFile(String file);
|
||||||
|
|
||||||
|
@Public
|
||||||
|
@Stable
|
||||||
|
public Path toPath() throws URISyntaxException {
|
||||||
|
String scheme = getScheme() == null ? "" : getScheme();
|
||||||
|
|
||||||
|
String authority = "";
|
||||||
|
if (getHost() != null) {
|
||||||
|
authority = getHost();
|
||||||
|
if (getUserInfo() != null) {
|
||||||
|
authority = getUserInfo() + "@" + authority;
|
||||||
|
}
|
||||||
|
if (getPort() > 0) {
|
||||||
|
authority += ":" + getPort();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return new Path(
|
||||||
|
(new URI(scheme, authority, getFile(), null, null)).normalize());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Public
|
||||||
|
@Stable
|
||||||
|
public static URL fromURI(URI uri) {
|
||||||
|
URL url =
|
||||||
|
RecordFactoryProvider.getRecordFactory(null).newRecordInstance(
|
||||||
|
URL.class);
|
||||||
|
if (uri.getHost() != null) {
|
||||||
|
url.setHost(uri.getHost());
|
||||||
|
}
|
||||||
|
if (uri.getUserInfo() != null) {
|
||||||
|
url.setUserInfo(uri.getUserInfo());
|
||||||
|
}
|
||||||
|
url.setPort(uri.getPort());
|
||||||
|
url.setScheme(uri.getScheme());
|
||||||
|
url.setFile(uri.getPath());
|
||||||
|
return url;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Public
|
||||||
|
@Stable
|
||||||
|
public static URL fromPath(Path path) {
|
||||||
|
return fromURI(path.toUri());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -412,13 +412,13 @@ public boolean init(String[] args) throws ParseException, IOException {
|
|||||||
if (!envs.containsKey(Environment.CONTAINER_ID.name())) {
|
if (!envs.containsKey(Environment.CONTAINER_ID.name())) {
|
||||||
if (cliParser.hasOption("app_attempt_id")) {
|
if (cliParser.hasOption("app_attempt_id")) {
|
||||||
String appIdStr = cliParser.getOptionValue("app_attempt_id", "");
|
String appIdStr = cliParser.getOptionValue("app_attempt_id", "");
|
||||||
appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
|
appAttemptID = ApplicationAttemptId.fromString(appIdStr);
|
||||||
} else {
|
} else {
|
||||||
throw new IllegalArgumentException(
|
throw new IllegalArgumentException(
|
||||||
"Application Attempt Id not set in the environment");
|
"Application Attempt Id not set in the environment");
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(envs
|
ContainerId containerId = ContainerId.fromString(envs
|
||||||
.get(Environment.CONTAINER_ID.name()));
|
.get(Environment.CONTAINER_ID.name()));
|
||||||
appAttemptID = containerId.getApplicationAttemptId();
|
appAttemptID = containerId.getApplicationAttemptId();
|
||||||
}
|
}
|
||||||
@ -1010,8 +1010,7 @@ public void run() {
|
|||||||
|
|
||||||
URL yarnUrl = null;
|
URL yarnUrl = null;
|
||||||
try {
|
try {
|
||||||
yarnUrl = ConverterUtils.getYarnUrlFromURI(
|
yarnUrl = URL.fromURI(new URI(renamedScriptPath.toString()));
|
||||||
new URI(renamedScriptPath.toString()));
|
|
||||||
} catch (URISyntaxException e) {
|
} catch (URISyntaxException e) {
|
||||||
LOG.error("Error when trying to use shell script path specified"
|
LOG.error("Error when trying to use shell script path specified"
|
||||||
+ " in env, path=" + renamedScriptPath, e);
|
+ " in env, path=" + renamedScriptPath, e);
|
||||||
|
@ -68,6 +68,7 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.QueueInfo;
|
import org.apache.hadoop.yarn.api.records.QueueInfo;
|
||||||
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
|
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
|
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
|
||||||
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
|
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
|
||||||
@ -822,7 +823,7 @@ private void addToLocalResources(FileSystem fs, String fileSrcPath,
|
|||||||
FileStatus scFileStatus = fs.getFileStatus(dst);
|
FileStatus scFileStatus = fs.getFileStatus(dst);
|
||||||
LocalResource scRsrc =
|
LocalResource scRsrc =
|
||||||
LocalResource.newInstance(
|
LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromURI(dst.toUri()),
|
URL.fromURI(dst.toUri()),
|
||||||
LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
|
LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
|
||||||
scFileStatus.getLen(), scFileStatus.getModificationTime());
|
scFileStatus.getLen(), scFileStatus.getModificationTime());
|
||||||
localResources.put(fileDstPath, scRsrc);
|
localResources.put(fileDstPath, scRsrc);
|
||||||
|
@ -53,7 +53,7 @@ public Set<TimelineEntityGroupId> getTimelineEntityGroupId(String entityType,
|
|||||||
public Set<TimelineEntityGroupId> getTimelineEntityGroupId(String entityId,
|
public Set<TimelineEntityGroupId> getTimelineEntityGroupId(String entityId,
|
||||||
String entityType) {
|
String entityType) {
|
||||||
if (ApplicationMaster.DSEntity.DS_CONTAINER.toString().equals(entityId)) {
|
if (ApplicationMaster.DSEntity.DS_CONTAINER.toString().equals(entityId)) {
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(entityId);
|
ContainerId containerId = ContainerId.fromString(entityId);
|
||||||
ApplicationId appId = containerId.getApplicationAttemptId()
|
ApplicationId appId = containerId.getApplicationAttemptId()
|
||||||
.getApplicationId();
|
.getApplicationId();
|
||||||
return toEntityGroupId(appId.toString());
|
return toEntityGroupId(appId.toString());
|
||||||
@ -69,7 +69,7 @@ public Set<TimelineEntityGroupId> getTimelineEntityGroupId(String entityType,
|
|||||||
}
|
}
|
||||||
|
|
||||||
private Set<TimelineEntityGroupId> toEntityGroupId(String strAppId) {
|
private Set<TimelineEntityGroupId> toEntityGroupId(String strAppId) {
|
||||||
ApplicationId appId = ConverterUtils.toApplicationId(strAppId);
|
ApplicationId appId = ApplicationId.fromString(strAppId);
|
||||||
TimelineEntityGroupId groupId = TimelineEntityGroupId.newInstance(
|
TimelineEntityGroupId groupId = TimelineEntityGroupId.newInstance(
|
||||||
appId, ApplicationMaster.CONTAINER_ENTITY_GROUP_ID);
|
appId, ApplicationMaster.CONTAINER_ENTITY_GROUP_ID);
|
||||||
Set<TimelineEntityGroupId> result = new HashSet<>();
|
Set<TimelineEntityGroupId> result = new HashSet<>();
|
||||||
|
@ -371,8 +371,8 @@ public void run() {
|
|||||||
}
|
}
|
||||||
String currAttemptEntityId
|
String currAttemptEntityId
|
||||||
= entitiesAttempts.getEntities().get(0).getEntityId();
|
= entitiesAttempts.getEntities().get(0).getEntityId();
|
||||||
ApplicationAttemptId attemptId
|
ApplicationAttemptId attemptId = ApplicationAttemptId.fromString(
|
||||||
= ConverterUtils.toApplicationAttemptId(currAttemptEntityId);
|
currAttemptEntityId);
|
||||||
NameValuePair primaryFilter = new NameValuePair(
|
NameValuePair primaryFilter = new NameValuePair(
|
||||||
ApplicationMaster.APPID_TIMELINE_FILTER_NAME,
|
ApplicationMaster.APPID_TIMELINE_FILTER_NAME,
|
||||||
attemptId.getApplicationId().toString());
|
attemptId.getApplicationId().toString());
|
||||||
|
@ -301,7 +301,7 @@ public int run(String[] args) throws Exception {
|
|||||||
*/
|
*/
|
||||||
private void signalToContainer(String containerIdStr,
|
private void signalToContainer(String containerIdStr,
|
||||||
SignalContainerCommand command) throws YarnException, IOException {
|
SignalContainerCommand command) throws YarnException, IOException {
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
sysout.println("Signalling container " + containerIdStr);
|
sysout.println("Signalling container " + containerIdStr);
|
||||||
client.signalToContainer(containerId, command);
|
client.signalToContainer(containerId, command);
|
||||||
}
|
}
|
||||||
@ -327,8 +327,8 @@ private int printApplicationAttemptReport(String applicationAttemptId)
|
|||||||
throws YarnException, IOException {
|
throws YarnException, IOException {
|
||||||
ApplicationAttemptReport appAttemptReport = null;
|
ApplicationAttemptReport appAttemptReport = null;
|
||||||
try {
|
try {
|
||||||
appAttemptReport = client.getApplicationAttemptReport(ConverterUtils
|
appAttemptReport = client.getApplicationAttemptReport(
|
||||||
.toApplicationAttemptId(applicationAttemptId));
|
ApplicationAttemptId.fromString(applicationAttemptId));
|
||||||
} catch (ApplicationNotFoundException e) {
|
} catch (ApplicationNotFoundException e) {
|
||||||
sysout.println("Application for AppAttempt with id '"
|
sysout.println("Application for AppAttempt with id '"
|
||||||
+ applicationAttemptId + "' doesn't exist in RM or Timeline Server.");
|
+ applicationAttemptId + "' doesn't exist in RM or Timeline Server.");
|
||||||
@ -384,8 +384,7 @@ private int printContainerReport(String containerId) throws YarnException,
|
|||||||
IOException {
|
IOException {
|
||||||
ContainerReport containerReport = null;
|
ContainerReport containerReport = null;
|
||||||
try {
|
try {
|
||||||
containerReport = client.getContainerReport((ConverterUtils
|
containerReport = client.getContainerReport(ContainerId.fromString(containerId));
|
||||||
.toContainerId(containerId)));
|
|
||||||
} catch (ApplicationNotFoundException e) {
|
} catch (ApplicationNotFoundException e) {
|
||||||
sysout.println("Application for Container with id '" + containerId
|
sysout.println("Application for Container with id '" + containerId
|
||||||
+ "' doesn't exist in RM or Timeline Server.");
|
+ "' doesn't exist in RM or Timeline Server.");
|
||||||
@ -515,7 +514,7 @@ private int killApplication(String[] applicationIds) throws YarnException,
|
|||||||
*/
|
*/
|
||||||
private void killApplication(String applicationId) throws YarnException,
|
private void killApplication(String applicationId) throws YarnException,
|
||||||
IOException {
|
IOException {
|
||||||
ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
|
ApplicationId appId = ApplicationId.fromString(applicationId);
|
||||||
ApplicationReport appReport = null;
|
ApplicationReport appReport = null;
|
||||||
try {
|
try {
|
||||||
appReport = client.getApplicationReport(appId);
|
appReport = client.getApplicationReport(appId);
|
||||||
@ -540,7 +539,7 @@ private void killApplication(String applicationId) throws YarnException,
|
|||||||
*/
|
*/
|
||||||
private void moveApplicationAcrossQueues(String applicationId, String queue)
|
private void moveApplicationAcrossQueues(String applicationId, String queue)
|
||||||
throws YarnException, IOException {
|
throws YarnException, IOException {
|
||||||
ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
|
ApplicationId appId = ApplicationId.fromString(applicationId);
|
||||||
ApplicationReport appReport = client.getApplicationReport(appId);
|
ApplicationReport appReport = client.getApplicationReport(appId);
|
||||||
if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
|
if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
|
||||||
|| appReport.getYarnApplicationState() == YarnApplicationState.KILLED
|
|| appReport.getYarnApplicationState() == YarnApplicationState.KILLED
|
||||||
@ -565,7 +564,7 @@ private void failApplicationAttempt(String attemptId) throws YarnException,
|
|||||||
IOException {
|
IOException {
|
||||||
ApplicationId appId;
|
ApplicationId appId;
|
||||||
ApplicationAttemptId attId;
|
ApplicationAttemptId attId;
|
||||||
attId = ConverterUtils.toApplicationAttemptId(attemptId);
|
attId = ApplicationAttemptId.fromString(attemptId);
|
||||||
appId = attId.getApplicationId();
|
appId = attId.getApplicationId();
|
||||||
|
|
||||||
sysout.println("Failing attempt " + attId + " of application " + appId);
|
sysout.println("Failing attempt " + attId + " of application " + appId);
|
||||||
@ -583,8 +582,8 @@ private int printApplicationReport(String applicationId)
|
|||||||
throws YarnException, IOException {
|
throws YarnException, IOException {
|
||||||
ApplicationReport appReport = null;
|
ApplicationReport appReport = null;
|
||||||
try {
|
try {
|
||||||
appReport = client.getApplicationReport(ConverterUtils
|
appReport = client.getApplicationReport(
|
||||||
.toApplicationId(applicationId));
|
ApplicationId.fromString(applicationId));
|
||||||
} catch (ApplicationNotFoundException e) {
|
} catch (ApplicationNotFoundException e) {
|
||||||
sysout.println("Application with id '" + applicationId
|
sysout.println("Application with id '" + applicationId
|
||||||
+ "' doesn't exist in RM or Timeline Server.");
|
+ "' doesn't exist in RM or Timeline Server.");
|
||||||
@ -684,7 +683,7 @@ private void listApplicationAttempts(String applicationId) throws YarnException,
|
|||||||
new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
|
new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
|
||||||
|
|
||||||
List<ApplicationAttemptReport> appAttemptsReport = client
|
List<ApplicationAttemptReport> appAttemptsReport = client
|
||||||
.getApplicationAttempts(ConverterUtils.toApplicationId(applicationId));
|
.getApplicationAttempts(ApplicationId.fromString(applicationId));
|
||||||
writer.println("Total number of application attempts " + ":"
|
writer.println("Total number of application attempts " + ":"
|
||||||
+ appAttemptsReport.size());
|
+ appAttemptsReport.size());
|
||||||
writer.printf(APPLICATION_ATTEMPTS_PATTERN, "ApplicationAttempt-Id",
|
writer.printf(APPLICATION_ATTEMPTS_PATTERN, "ApplicationAttempt-Id",
|
||||||
@ -711,8 +710,8 @@ private void listContainers(String appAttemptId) throws YarnException,
|
|||||||
PrintWriter writer = new PrintWriter(
|
PrintWriter writer = new PrintWriter(
|
||||||
new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
|
new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
|
||||||
|
|
||||||
List<ContainerReport> appsReport = client
|
List<ContainerReport> appsReport = client.getContainers(
|
||||||
.getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId));
|
ApplicationAttemptId.fromString(appAttemptId));
|
||||||
writer.println("Total number of containers " + ":" + appsReport.size());
|
writer.println("Total number of containers " + ":" + appsReport.size());
|
||||||
writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
|
writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
|
||||||
"Finish Time", "State", "Host", "Node Http Address", "LOG-URL");
|
"Finish Time", "State", "Host", "Node Http Address", "LOG-URL");
|
||||||
@ -735,7 +734,7 @@ private void listContainers(String appAttemptId) throws YarnException,
|
|||||||
*/
|
*/
|
||||||
private void updateApplicationPriority(String applicationId, String priority)
|
private void updateApplicationPriority(String applicationId, String priority)
|
||||||
throws YarnException, IOException {
|
throws YarnException, IOException {
|
||||||
ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
|
ApplicationId appId = ApplicationId.fromString(applicationId);
|
||||||
Priority newAppPriority = Priority.newInstance(Integer.parseInt(priority));
|
Priority newAppPriority = Priority.newInstance(Integer.parseInt(priority));
|
||||||
sysout.println("Updating priority of an application " + applicationId);
|
sysout.println("Updating priority of an application " + applicationId);
|
||||||
client.updateApplicationPriority(appId, newAppPriority);
|
client.updateApplicationPriority(appId, newAppPriority);
|
||||||
|
@ -45,6 +45,7 @@
|
|||||||
import org.apache.hadoop.util.Tool;
|
import org.apache.hadoop.util.Tool;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
import org.apache.hadoop.yarn.api.records.ContainerReport;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||||
import org.apache.hadoop.yarn.client.api.YarnClient;
|
import org.apache.hadoop.yarn.client.api.YarnClient;
|
||||||
@ -199,7 +200,7 @@ public int run(String[] args) throws Exception {
|
|||||||
|
|
||||||
ApplicationId appId = null;
|
ApplicationId appId = null;
|
||||||
try {
|
try {
|
||||||
appId = ConverterUtils.toApplicationId(appIdStr);
|
appId = ApplicationId.fromString(appIdStr);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
System.err.println("Invalid ApplicationId specified");
|
System.err.println("Invalid ApplicationId specified");
|
||||||
return -1;
|
return -1;
|
||||||
@ -523,8 +524,8 @@ private ContainerReport getContainerReport(String containerIdStr)
|
|||||||
throws YarnException, IOException {
|
throws YarnException, IOException {
|
||||||
YarnClient yarnClient = createYarnClient();
|
YarnClient yarnClient = createYarnClient();
|
||||||
try {
|
try {
|
||||||
return yarnClient.getContainerReport(ConverterUtils
|
return yarnClient.getContainerReport(
|
||||||
.toContainerId(containerIdStr));
|
ContainerId.fromString(containerIdStr));
|
||||||
} finally {
|
} finally {
|
||||||
yarnClient.close();
|
yarnClient.close();
|
||||||
}
|
}
|
||||||
|
@ -243,7 +243,7 @@ private void listDetailedClusterNodes(Set<NodeState> nodeStates)
|
|||||||
*/
|
*/
|
||||||
private void printNodeStatus(String nodeIdStr) throws YarnException,
|
private void printNodeStatus(String nodeIdStr) throws YarnException,
|
||||||
IOException {
|
IOException {
|
||||||
NodeId nodeId = ConverterUtils.toNodeId(nodeIdStr);
|
NodeId nodeId = NodeId.fromString(nodeIdStr);
|
||||||
List<NodeReport> nodesReport = client.getNodeReports();
|
List<NodeReport> nodesReport = client.getNodeReports();
|
||||||
// Use PrintWriter.println, which uses correct platform line ending.
|
// Use PrintWriter.println, which uses correct platform line ending.
|
||||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||||
|
@ -427,7 +427,7 @@ private int updateNodeResource(String nodeIdStr, int memSize,
|
|||||||
ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol();
|
ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol();
|
||||||
UpdateNodeResourceRequest request =
|
UpdateNodeResourceRequest request =
|
||||||
recordFactory.newRecordInstance(UpdateNodeResourceRequest.class);
|
recordFactory.newRecordInstance(UpdateNodeResourceRequest.class);
|
||||||
NodeId nodeId = ConverterUtils.toNodeId(nodeIdStr);
|
NodeId nodeId = NodeId.fromString(nodeIdStr);
|
||||||
|
|
||||||
Resource resource = Resources.createResource(memSize, cores);
|
Resource resource = Resources.createResource(memSize, cores);
|
||||||
Map<NodeId, ResourceOption> resourceMap =
|
Map<NodeId, ResourceOption> resourceMap =
|
||||||
|
@ -222,7 +222,7 @@ public void testUpdateNodeResource() throws Exception {
|
|||||||
verify(admin).updateNodeResource(argument.capture());
|
verify(admin).updateNodeResource(argument.capture());
|
||||||
UpdateNodeResourceRequest request = argument.getValue();
|
UpdateNodeResourceRequest request = argument.getValue();
|
||||||
Map<NodeId, ResourceOption> resourceMap = request.getNodeResourceMap();
|
Map<NodeId, ResourceOption> resourceMap = request.getNodeResourceMap();
|
||||||
NodeId nodeId = ConverterUtils.toNodeId(nodeIdStr);
|
NodeId nodeId = NodeId.fromString(nodeIdStr);
|
||||||
Resource expectedResource = Resources.createResource(memSize, cores);
|
Resource expectedResource = Resources.createResource(memSize, cores);
|
||||||
ResourceOption resource = resourceMap.get(nodeId);
|
ResourceOption resource = resourceMap.get(nodeId);
|
||||||
assertNotNull("resource for " + nodeIdStr + " shouldn't be null.",
|
assertNotNull("resource for " + nodeIdStr + " shouldn't be null.",
|
||||||
|
@ -99,7 +99,7 @@ private static void deleteOldLogDirsFrom(Path dir, long cutoffMillis,
|
|||||||
if(appDir.isDirectory() &&
|
if(appDir.isDirectory() &&
|
||||||
appDir.getModificationTime() < cutoffMillis) {
|
appDir.getModificationTime() < cutoffMillis) {
|
||||||
boolean appTerminated =
|
boolean appTerminated =
|
||||||
isApplicationTerminated(ConverterUtils.toApplicationId(appDir
|
isApplicationTerminated(ApplicationId.fromString(appDir
|
||||||
.getPath().getName()), rmClient);
|
.getPath().getName()), rmClient);
|
||||||
if(appTerminated && shouldDeleteLogDir(appDir, cutoffMillis, fs)) {
|
if(appTerminated && shouldDeleteLogDir(appDir, cutoffMillis, fs)) {
|
||||||
try {
|
try {
|
||||||
|
@ -193,14 +193,11 @@ public LogValue(List<String> rootLogDirs, ContainerId containerId,
|
|||||||
private Set<File> getPendingLogFilesToUploadForThisContainer() {
|
private Set<File> getPendingLogFilesToUploadForThisContainer() {
|
||||||
Set<File> pendingUploadFiles = new HashSet<File>();
|
Set<File> pendingUploadFiles = new HashSet<File>();
|
||||||
for (String rootLogDir : this.rootLogDirs) {
|
for (String rootLogDir : this.rootLogDirs) {
|
||||||
File appLogDir =
|
File appLogDir = new File(rootLogDir,
|
||||||
new File(rootLogDir,
|
|
||||||
ConverterUtils.toString(
|
|
||||||
this.containerId.getApplicationAttemptId().
|
this.containerId.getApplicationAttemptId().
|
||||||
getApplicationId())
|
getApplicationId().toString());
|
||||||
);
|
|
||||||
File containerLogDir =
|
File containerLogDir =
|
||||||
new File(appLogDir, ConverterUtils.toString(this.containerId));
|
new File(appLogDir, this.containerId.toString());
|
||||||
|
|
||||||
if (!containerLogDir.isDirectory()) {
|
if (!containerLogDir.isDirectory()) {
|
||||||
continue; // ContainerDir may have been deleted by the user.
|
continue; // ContainerDir may have been deleted by the user.
|
||||||
|
@ -62,7 +62,7 @@ public int dumpAContainersLogsForALogType(String appId, String containerId,
|
|||||||
YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
|
YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
|
||||||
YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
|
YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
|
||||||
String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf());
|
String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf());
|
||||||
ApplicationId applicationId = ConverterUtils.toApplicationId(appId);
|
ApplicationId applicationId = ApplicationId.fromString(appId);
|
||||||
Path remoteAppLogDir = LogAggregationUtils.getRemoteAppLogDir(
|
Path remoteAppLogDir = LogAggregationUtils.getRemoteAppLogDir(
|
||||||
remoteRootLogDir, applicationId, jobOwner,
|
remoteRootLogDir, applicationId, jobOwner,
|
||||||
suffix);
|
suffix);
|
||||||
|
@ -18,18 +18,13 @@
|
|||||||
|
|
||||||
package org.apache.hadoop.yarn.util;
|
package org.apache.hadoop.yarn.util;
|
||||||
|
|
||||||
import static org.apache.hadoop.yarn.util.StringHelper._split;
|
|
||||||
|
|
||||||
import java.net.InetSocketAddress;
|
import java.net.InetSocketAddress;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.URISyntaxException;
|
import java.net.URISyntaxException;
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Iterator;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Map.Entry;
|
|
||||||
import java.util.NoSuchElementException;
|
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.io.Text;
|
import org.apache.hadoop.io.Text;
|
||||||
import org.apache.hadoop.security.SecurityUtil;
|
import org.apache.hadoop.security.SecurityUtil;
|
||||||
@ -41,7 +36,6 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.api.records.URL;
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -49,7 +43,7 @@
|
|||||||
* from/to 'serializableFormat' to/from hadoop/nativejava data structures.
|
* from/to 'serializableFormat' to/from hadoop/nativejava data structures.
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
@Private
|
@Public
|
||||||
public class ConverterUtils {
|
public class ConverterUtils {
|
||||||
|
|
||||||
public static final String APPLICATION_PREFIX = "application";
|
public static final String APPLICATION_PREFIX = "application";
|
||||||
@ -58,174 +52,114 @@ public class ConverterUtils {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* return a hadoop path from a given url
|
* return a hadoop path from a given url
|
||||||
|
* This method is deprecated, use {@link URL#toPath()} instead.
|
||||||
*
|
*
|
||||||
* @param url
|
* @param url
|
||||||
* url to convert
|
* url to convert
|
||||||
* @return path from {@link URL}
|
* @return path from {@link URL}
|
||||||
* @throws URISyntaxException
|
* @throws URISyntaxException
|
||||||
*/
|
*/
|
||||||
|
@Public
|
||||||
|
@Deprecated
|
||||||
public static Path getPathFromYarnURL(URL url) throws URISyntaxException {
|
public static Path getPathFromYarnURL(URL url) throws URISyntaxException {
|
||||||
String scheme = url.getScheme() == null ? "" : url.getScheme();
|
return url.toPath();
|
||||||
|
|
||||||
String authority = "";
|
|
||||||
if (url.getHost() != null) {
|
|
||||||
authority = url.getHost();
|
|
||||||
if (url.getUserInfo() != null) {
|
|
||||||
authority = url.getUserInfo() + "@" + authority;
|
|
||||||
}
|
|
||||||
if (url.getPort() > 0) {
|
|
||||||
authority += ":" + url.getPort();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return new Path(
|
/*
|
||||||
(new URI(scheme, authority, url.getFile(), null, null)).normalize());
|
* This method is deprecated, use {@link URL#fromPath(Path)} instead.
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* change from CharSequence to string for map key and value
|
|
||||||
* @param env map for converting
|
|
||||||
* @return string,string map
|
|
||||||
*/
|
*/
|
||||||
public static Map<String, String> convertToString(
|
@Public
|
||||||
Map<CharSequence, CharSequence> env) {
|
@Deprecated
|
||||||
|
|
||||||
Map<String, String> stringMap = new HashMap<String, String>();
|
|
||||||
for (Entry<CharSequence, CharSequence> entry: env.entrySet()) {
|
|
||||||
stringMap.put(entry.getKey().toString(), entry.getValue().toString());
|
|
||||||
}
|
|
||||||
return stringMap;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static URL getYarnUrlFromPath(Path path) {
|
public static URL getYarnUrlFromPath(Path path) {
|
||||||
return getYarnUrlFromURI(path.toUri());
|
return URL.fromPath(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This method is deprecated, use {@link URL#fromURI(URI)} instead.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Deprecated
|
||||||
public static URL getYarnUrlFromURI(URI uri) {
|
public static URL getYarnUrlFromURI(URI uri) {
|
||||||
URL url = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(URL.class);
|
return URL.fromURI(uri);
|
||||||
if (uri.getHost() != null) {
|
|
||||||
url.setHost(uri.getHost());
|
|
||||||
}
|
|
||||||
if (uri.getUserInfo() != null) {
|
|
||||||
url.setUserInfo(uri.getUserInfo());
|
|
||||||
}
|
|
||||||
url.setPort(uri.getPort());
|
|
||||||
url.setScheme(uri.getScheme());
|
|
||||||
url.setFile(uri.getPath());
|
|
||||||
return url;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This method is deprecated, use {@link ApplicationId#toString()} instead.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Deprecated
|
||||||
public static String toString(ApplicationId appId) {
|
public static String toString(ApplicationId appId) {
|
||||||
return appId.toString();
|
return appId.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This method is deprecated, use {@link ApplicationId#fromString(String)}
|
||||||
|
* instead.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Deprecated
|
||||||
public static ApplicationId toApplicationId(RecordFactory recordFactory,
|
public static ApplicationId toApplicationId(RecordFactory recordFactory,
|
||||||
String appIdStr) {
|
String applicationIdStr) {
|
||||||
Iterator<String> it = _split(appIdStr).iterator();
|
return ApplicationId.fromString(applicationIdStr);
|
||||||
if (!it.next().equals(APPLICATION_PREFIX)) {
|
|
||||||
throw new IllegalArgumentException("Invalid ApplicationId prefix: "
|
|
||||||
+ appIdStr + ". The valid ApplicationId should start with prefix "
|
|
||||||
+ APPLICATION_PREFIX);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return toApplicationId(recordFactory, it);
|
|
||||||
} catch (NumberFormatException n) {
|
|
||||||
throw new IllegalArgumentException("Invalid ApplicationId: " + appIdStr,
|
|
||||||
n);
|
|
||||||
} catch (NoSuchElementException e) {
|
|
||||||
throw new IllegalArgumentException("Invalid ApplicationId: " + appIdStr,
|
|
||||||
e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static ApplicationId toApplicationId(RecordFactory recordFactory,
|
|
||||||
Iterator<String> it) {
|
|
||||||
ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
|
|
||||||
Integer.parseInt(it.next()));
|
|
||||||
return appId;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static ApplicationAttemptId toApplicationAttemptId(
|
|
||||||
Iterator<String> it) throws NumberFormatException {
|
|
||||||
ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
|
|
||||||
Integer.parseInt(it.next()));
|
|
||||||
ApplicationAttemptId appAttemptId =
|
|
||||||
ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next()));
|
|
||||||
return appAttemptId;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static ApplicationId toApplicationId(
|
|
||||||
Iterator<String> it) throws NumberFormatException {
|
|
||||||
ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
|
|
||||||
Integer.parseInt(it.next()));
|
|
||||||
return appId;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This method is deprecated, use {@link ContainerId#toString()} instead.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Deprecated
|
||||||
public static String toString(ContainerId cId) {
|
public static String toString(ContainerId cId) {
|
||||||
return cId == null ? null : cId.toString();
|
return cId == null ? null : cId.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Private
|
||||||
|
@InterfaceStability.Unstable
|
||||||
public static NodeId toNodeIdWithDefaultPort(String nodeIdStr) {
|
public static NodeId toNodeIdWithDefaultPort(String nodeIdStr) {
|
||||||
if (nodeIdStr.indexOf(":") < 0) {
|
if (nodeIdStr.indexOf(":") < 0) {
|
||||||
return toNodeId(nodeIdStr + ":0");
|
return NodeId.fromString(nodeIdStr + ":0");
|
||||||
}
|
}
|
||||||
return toNodeId(nodeIdStr);
|
return NodeId.fromString(nodeIdStr);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This method is deprecated, use {@link NodeId#fromString(String)} instead.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Deprecated
|
||||||
public static NodeId toNodeId(String nodeIdStr) {
|
public static NodeId toNodeId(String nodeIdStr) {
|
||||||
String[] parts = nodeIdStr.split(":");
|
return NodeId.fromString(nodeIdStr);
|
||||||
if (parts.length != 2) {
|
|
||||||
throw new IllegalArgumentException("Invalid NodeId [" + nodeIdStr
|
|
||||||
+ "]. Expected host:port");
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
NodeId nodeId =
|
|
||||||
NodeId.newInstance(parts[0].trim(), Integer.parseInt(parts[1]));
|
|
||||||
return nodeId;
|
|
||||||
} catch (NumberFormatException e) {
|
|
||||||
throw new IllegalArgumentException("Invalid port: " + parts[1], e);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This method is deprecated, use {@link ContainerId#fromString(String)}
|
||||||
|
* instead.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Deprecated
|
||||||
public static ContainerId toContainerId(String containerIdStr) {
|
public static ContainerId toContainerId(String containerIdStr) {
|
||||||
return ContainerId.fromString(containerIdStr);
|
return ContainerId.fromString(containerIdStr);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This method is deprecated, use {@link ApplicationAttemptId#toString()}
|
||||||
|
* instead.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Deprecated
|
||||||
public static ApplicationAttemptId toApplicationAttemptId(
|
public static ApplicationAttemptId toApplicationAttemptId(
|
||||||
String applicationAttmeptIdStr) {
|
String applicationAttemptIdStr) {
|
||||||
Iterator<String> it = _split(applicationAttmeptIdStr).iterator();
|
return ApplicationAttemptId.fromString(applicationAttemptIdStr);
|
||||||
if (!it.next().equals(APPLICATION_ATTEMPT_PREFIX)) {
|
|
||||||
throw new IllegalArgumentException("Invalid AppAttemptId prefix: "
|
|
||||||
+ applicationAttmeptIdStr);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return toApplicationAttemptId(it);
|
|
||||||
} catch (NumberFormatException n) {
|
|
||||||
throw new IllegalArgumentException("Invalid AppAttemptId: "
|
|
||||||
+ applicationAttmeptIdStr, n);
|
|
||||||
} catch (NoSuchElementException e) {
|
|
||||||
throw new IllegalArgumentException("Invalid AppAttemptId: "
|
|
||||||
+ applicationAttmeptIdStr, e);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* This method is deprecated, use {@link ApplicationId#fromString(String)}
|
||||||
|
* instead.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Deprecated
|
||||||
public static ApplicationId toApplicationId(
|
public static ApplicationId toApplicationId(
|
||||||
String appIdStr) {
|
String appIdStr) {
|
||||||
Iterator<String> it = _split(appIdStr).iterator();
|
return ApplicationId.fromString(appIdStr);
|
||||||
if (!it.next().equals(APPLICATION_PREFIX)) {
|
|
||||||
throw new IllegalArgumentException("Invalid ApplicationId prefix: "
|
|
||||||
+ appIdStr + ". The valid ApplicationId should start with prefix "
|
|
||||||
+ APPLICATION_PREFIX);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
return toApplicationId(it);
|
|
||||||
} catch (NumberFormatException n) {
|
|
||||||
throw new IllegalArgumentException("Invalid ApplicationId: "
|
|
||||||
+ appIdStr, n);
|
|
||||||
} catch (NoSuchElementException e) {
|
|
||||||
throw new IllegalArgumentException("Invalid ApplicationId: "
|
|
||||||
+ appIdStr, e);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -346,7 +346,7 @@ private long unpack(File localrsrc, File dst) throws IOException {
|
|||||||
public Path call() throws Exception {
|
public Path call() throws Exception {
|
||||||
final Path sCopy;
|
final Path sCopy;
|
||||||
try {
|
try {
|
||||||
sCopy = ConverterUtils.getPathFromYarnURL(resource.getResource());
|
sCopy = resource.getResource().toPath();
|
||||||
} catch (URISyntaxException e) {
|
} catch (URISyntaxException e) {
|
||||||
throw new IOException("Invalid resource", e);
|
throw new IOException("Invalid resource", e);
|
||||||
}
|
}
|
||||||
|
@ -279,7 +279,7 @@ private ContainerId verifyAndGetContainerId(Block html) {
|
|||||||
}
|
}
|
||||||
ContainerId containerId = null;
|
ContainerId containerId = null;
|
||||||
try {
|
try {
|
||||||
containerId = ConverterUtils.toContainerId(containerIdStr);
|
containerId = ContainerId.fromString(containerIdStr);
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
html.h1()
|
html.h1()
|
||||||
._("Cannot get container logs for invalid containerId: "
|
._("Cannot get container logs for invalid containerId: "
|
||||||
@ -297,7 +297,7 @@ private NodeId verifyAndGetNodeId(Block html) {
|
|||||||
}
|
}
|
||||||
NodeId nodeId = null;
|
NodeId nodeId = null;
|
||||||
try {
|
try {
|
||||||
nodeId = ConverterUtils.toNodeId(nodeIdStr);
|
nodeId = NodeId.fromString(nodeIdStr);
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
html.h1()._("Cannot get container logs. Invalid nodeId: " + nodeIdStr)
|
html.h1()._("Cannot get container logs. Invalid nodeId: " + nodeIdStr)
|
||||||
._();
|
._();
|
||||||
|
@ -391,7 +391,7 @@ public static ApplicationId parseApplicationId(RecordFactory recordFactory,
|
|||||||
}
|
}
|
||||||
ApplicationId aid = null;
|
ApplicationId aid = null;
|
||||||
try {
|
try {
|
||||||
aid = ConverterUtils.toApplicationId(recordFactory, appId);
|
aid = ApplicationId.fromString(appId);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new BadRequestException(e);
|
throw new BadRequestException(e);
|
||||||
}
|
}
|
||||||
|
@ -34,55 +34,56 @@ public class TestConverterUtils {
|
|||||||
@Test
|
@Test
|
||||||
public void testConvertUrlWithNoPort() throws URISyntaxException {
|
public void testConvertUrlWithNoPort() throws URISyntaxException {
|
||||||
Path expectedPath = new Path("hdfs://foo.com");
|
Path expectedPath = new Path("hdfs://foo.com");
|
||||||
URL url = ConverterUtils.getYarnUrlFromPath(expectedPath);
|
URL url = URL.fromPath(expectedPath);
|
||||||
Path actualPath = ConverterUtils.getPathFromYarnURL(url);
|
Path actualPath = url.toPath();
|
||||||
assertEquals(expectedPath, actualPath);
|
assertEquals(expectedPath, actualPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testConvertUrlWithUserinfo() throws URISyntaxException {
|
public void testConvertUrlWithUserinfo() throws URISyntaxException {
|
||||||
Path expectedPath = new Path("foo://username:password@example.com:8042");
|
Path expectedPath = new Path("foo://username:password@example.com:8042");
|
||||||
URL url = ConverterUtils.getYarnUrlFromPath(expectedPath);
|
URL url = URL.fromPath(expectedPath);
|
||||||
Path actualPath = ConverterUtils.getPathFromYarnURL(url);
|
Path actualPath = url.toPath();
|
||||||
assertEquals(expectedPath, actualPath);
|
assertEquals(expectedPath, actualPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testContainerId() throws URISyntaxException {
|
public void testContainerId() throws URISyntaxException {
|
||||||
ContainerId id = TestContainerId.newContainerId(0, 0, 0, 0);
|
ContainerId id = TestContainerId.newContainerId(0, 0, 0, 0);
|
||||||
String cid = ConverterUtils.toString(id);
|
String cid = id.toString();
|
||||||
assertEquals("container_0_0000_00_000000", cid);
|
assertEquals("container_0_0000_00_000000", cid);
|
||||||
ContainerId gen = ConverterUtils.toContainerId(cid);
|
ContainerId gen = ContainerId.fromString(cid);
|
||||||
assertEquals(gen, id);
|
assertEquals(gen, id);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testContainerIdWithEpoch() throws URISyntaxException {
|
public void testContainerIdWithEpoch() throws URISyntaxException {
|
||||||
ContainerId id = TestContainerId.newContainerId(0, 0, 0, 25645811);
|
ContainerId id = TestContainerId.newContainerId(0, 0, 0, 25645811);
|
||||||
String cid = ConverterUtils.toString(id);
|
String cid = id.toString();
|
||||||
assertEquals("container_0_0000_00_25645811", cid);
|
assertEquals("container_0_0000_00_25645811", cid);
|
||||||
ContainerId gen = ConverterUtils.toContainerId(cid);
|
ContainerId gen = ContainerId.fromString(cid);
|
||||||
assertEquals(gen.toString(), id.toString());
|
assertEquals(gen.toString(), id.toString());
|
||||||
|
|
||||||
long ts = System.currentTimeMillis();
|
long ts = System.currentTimeMillis();
|
||||||
ContainerId id2 =
|
ContainerId id2 =
|
||||||
TestContainerId.newContainerId(36473, 4365472, ts, 4298334883325L);
|
TestContainerId.newContainerId(36473, 4365472, ts, 4298334883325L);
|
||||||
String cid2 = ConverterUtils.toString(id2);
|
String cid2 = id2.toString();
|
||||||
assertEquals(
|
assertEquals(
|
||||||
"container_e03_" + ts + "_36473_4365472_999799999997", cid2);
|
"container_e03_" + ts + "_36473_4365472_999799999997", cid2);
|
||||||
ContainerId gen2 = ConverterUtils.toContainerId(cid2);
|
ContainerId gen2 = ContainerId.fromString(cid2);
|
||||||
assertEquals(gen2.toString(), id2.toString());
|
assertEquals(gen2.toString(), id2.toString());
|
||||||
|
|
||||||
ContainerId id3 =
|
ContainerId id3 =
|
||||||
TestContainerId.newContainerId(36473, 4365472, ts, 844424930131965L);
|
TestContainerId.newContainerId(36473, 4365472, ts, 844424930131965L);
|
||||||
String cid3 = ConverterUtils.toString(id3);
|
String cid3 = id3.toString();
|
||||||
assertEquals(
|
assertEquals(
|
||||||
"container_e767_" + ts + "_36473_4365472_1099511627773", cid3);
|
"container_e767_" + ts + "_36473_4365472_1099511627773", cid3);
|
||||||
ContainerId gen3 = ConverterUtils.toContainerId(cid3);
|
ContainerId gen3 = ContainerId.fromString(cid3);
|
||||||
assertEquals(gen3.toString(), id3.toString());
|
assertEquals(gen3.toString(), id3.toString());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@SuppressWarnings("deprecation")
|
||||||
public void testContainerIdNull() throws URISyntaxException {
|
public void testContainerIdNull() throws URISyntaxException {
|
||||||
assertNull(ConverterUtils.toString((ContainerId)null));
|
assertNull(ConverterUtils.toString((ContainerId)null));
|
||||||
}
|
}
|
||||||
@ -101,16 +102,19 @@ public void testNodeIdWithDefaultPort() throws URISyntaxException {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test(expected = IllegalArgumentException.class)
|
@Test(expected = IllegalArgumentException.class)
|
||||||
|
@SuppressWarnings("deprecation")
|
||||||
public void testInvalidContainerId() {
|
public void testInvalidContainerId() {
|
||||||
ConverterUtils.toContainerId("container_e20_1423221031460_0003_01");
|
ContainerId.fromString("container_e20_1423221031460_0003_01");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(expected = IllegalArgumentException.class)
|
@Test(expected = IllegalArgumentException.class)
|
||||||
|
@SuppressWarnings("deprecation")
|
||||||
public void testInvalidAppattemptId() {
|
public void testInvalidAppattemptId() {
|
||||||
ConverterUtils.toApplicationAttemptId("appattempt_1423221031460");
|
ConverterUtils.toApplicationAttemptId("appattempt_1423221031460");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(expected = IllegalArgumentException.class)
|
@Test(expected = IllegalArgumentException.class)
|
||||||
|
@SuppressWarnings("deprecation")
|
||||||
public void testApplicationId() {
|
public void testApplicationId() {
|
||||||
ConverterUtils.toApplicationId("application_1423221031460");
|
ConverterUtils.toApplicationId("application_1423221031460");
|
||||||
}
|
}
|
||||||
|
@ -18,12 +18,32 @@
|
|||||||
|
|
||||||
package org.apache.hadoop.yarn.util;
|
package org.apache.hadoop.yarn.util;
|
||||||
|
|
||||||
import static org.apache.hadoop.fs.CreateFlag.CREATE;
|
import com.google.common.cache.CacheBuilder;
|
||||||
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
|
import com.google.common.cache.CacheLoader;
|
||||||
import static org.junit.Assert.assertEquals;
|
import com.google.common.cache.LoadingCache;
|
||||||
import static org.junit.Assert.assertSame;
|
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
|
||||||
import static org.junit.Assert.assertTrue;
|
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
|
||||||
import static org.junit.Assume.assumeTrue;
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.CommonConfigurationKeys;
|
||||||
|
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||||
|
import org.apache.hadoop.fs.FileContext;
|
||||||
|
import org.apache.hadoop.fs.FileStatus;
|
||||||
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
|
import org.apache.hadoop.fs.LocalDirAllocator;
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
|
import org.apache.hadoop.yarn.api.records.LocalResource;
|
||||||
|
import org.apache.hadoop.yarn.api.records.LocalResourceType;
|
||||||
|
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
||||||
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
|
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileOutputStream;
|
import java.io.FileOutputStream;
|
||||||
@ -53,33 +73,12 @@
|
|||||||
import java.util.zip.ZipEntry;
|
import java.util.zip.ZipEntry;
|
||||||
import java.util.zip.ZipOutputStream;
|
import java.util.zip.ZipOutputStream;
|
||||||
|
|
||||||
import org.junit.Assert;
|
import static org.apache.hadoop.fs.CreateFlag.CREATE;
|
||||||
|
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
|
||||||
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
|
import static org.junit.Assert.assertEquals;
|
||||||
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
|
import static org.junit.Assert.assertSame;
|
||||||
import org.apache.commons.logging.Log;
|
import static org.junit.Assert.assertTrue;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import static org.junit.Assume.assumeTrue;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
|
||||||
import org.apache.hadoop.fs.CommonConfigurationKeys;
|
|
||||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
|
||||||
import org.apache.hadoop.fs.FileContext;
|
|
||||||
import org.apache.hadoop.fs.FileStatus;
|
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
|
||||||
import org.apache.hadoop.fs.LocalDirAllocator;
|
|
||||||
import org.apache.hadoop.fs.Path;
|
|
||||||
import org.apache.hadoop.fs.permission.FsPermission;
|
|
||||||
import org.apache.hadoop.security.UserGroupInformation;
|
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResource;
|
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResourceType;
|
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
|
||||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
|
||||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
|
||||||
import org.junit.AfterClass;
|
|
||||||
import org.junit.Test;
|
|
||||||
|
|
||||||
import com.google.common.cache.CacheBuilder;
|
|
||||||
import com.google.common.cache.CacheLoader;
|
|
||||||
import com.google.common.cache.LoadingCache;
|
|
||||||
|
|
||||||
public class TestFSDownload {
|
public class TestFSDownload {
|
||||||
|
|
||||||
@ -103,7 +102,7 @@ static LocalResource createFile(FileContext files, Path p, int len,
|
|||||||
Random r, LocalResourceVisibility vis) throws IOException {
|
Random r, LocalResourceVisibility vis) throws IOException {
|
||||||
createFile(files, p, len, r);
|
createFile(files, p, len, r);
|
||||||
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
|
ret.setResource(URL.fromPath(p));
|
||||||
ret.setSize(len);
|
ret.setSize(len);
|
||||||
ret.setType(LocalResourceType.FILE);
|
ret.setType(LocalResourceType.FILE);
|
||||||
ret.setVisibility(vis);
|
ret.setVisibility(vis);
|
||||||
@ -134,7 +133,7 @@ static LocalResource createJar(FileContext files, Path p,
|
|||||||
LOG.info("Done writing jar stream ");
|
LOG.info("Done writing jar stream ");
|
||||||
out.close();
|
out.close();
|
||||||
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
|
ret.setResource(URL.fromPath(p));
|
||||||
FileStatus status = files.getFileStatus(p);
|
FileStatus status = files.getFileStatus(p);
|
||||||
ret.setSize(status.getLen());
|
ret.setSize(status.getLen());
|
||||||
ret.setTimestamp(status.getModificationTime());
|
ret.setTimestamp(status.getModificationTime());
|
||||||
@ -162,7 +161,7 @@ static LocalResource createTarFile(FileContext files, Path p, int len,
|
|||||||
out.close();
|
out.close();
|
||||||
|
|
||||||
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
|
ret.setResource(URL.fromPath(new Path(p.toString()
|
||||||
+ ".tar")));
|
+ ".tar")));
|
||||||
ret.setSize(len);
|
ret.setSize(len);
|
||||||
ret.setType(LocalResourceType.ARCHIVE);
|
ret.setType(LocalResourceType.ARCHIVE);
|
||||||
@ -190,7 +189,7 @@ static LocalResource createTgzFile(FileContext files, Path p, int len,
|
|||||||
out.close();
|
out.close();
|
||||||
|
|
||||||
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
|
ret.setResource(URL.fromPath(new Path(p.toString()
|
||||||
+ ".tar.gz")));
|
+ ".tar.gz")));
|
||||||
ret.setSize(len);
|
ret.setSize(len);
|
||||||
ret.setType(LocalResourceType.ARCHIVE);
|
ret.setType(LocalResourceType.ARCHIVE);
|
||||||
@ -216,7 +215,7 @@ static LocalResource createJarFile(FileContext files, Path p, int len,
|
|||||||
out.close();
|
out.close();
|
||||||
|
|
||||||
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
|
ret.setResource(URL.fromPath(new Path(p.toString()
|
||||||
+ ".jar")));
|
+ ".jar")));
|
||||||
ret.setSize(len);
|
ret.setSize(len);
|
||||||
ret.setType(LocalResourceType.ARCHIVE);
|
ret.setType(LocalResourceType.ARCHIVE);
|
||||||
@ -242,7 +241,7 @@ static LocalResource createZipFile(FileContext files, Path p, int len,
|
|||||||
out.close();
|
out.close();
|
||||||
|
|
||||||
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
|
ret.setResource(URL.fromPath(new Path(p.toString()
|
||||||
+ ".ZIP")));
|
+ ".ZIP")));
|
||||||
ret.setSize(len);
|
ret.setSize(len);
|
||||||
ret.setType(LocalResourceType.ARCHIVE);
|
ret.setType(LocalResourceType.ARCHIVE);
|
||||||
|
@ -278,7 +278,7 @@ private static ApplicationReportExt convertToApplicationReport(
|
|||||||
}
|
}
|
||||||
if (field == ApplicationReportField.USER_AND_ACLS) {
|
if (field == ApplicationReportField.USER_AND_ACLS) {
|
||||||
return new ApplicationReportExt(ApplicationReport.newInstance(
|
return new ApplicationReportExt(ApplicationReport.newInstance(
|
||||||
ConverterUtils.toApplicationId(entity.getEntityId()),
|
ApplicationId.fromString(entity.getEntityId()),
|
||||||
latestApplicationAttemptId, user, queue, name, null, -1, null,
|
latestApplicationAttemptId, user, queue, name, null, -1, null,
|
||||||
state, diagnosticsInfo, null, createdTime, finishedTime,
|
state, diagnosticsInfo, null, createdTime, finishedTime,
|
||||||
finalStatus, null, null, progress, type, null, appTags,
|
finalStatus, null, null, progress, type, null, appTags,
|
||||||
@ -394,11 +394,8 @@ private static ApplicationReportExt convertToApplicationReport(
|
|||||||
}
|
}
|
||||||
if (eventInfo
|
if (eventInfo
|
||||||
.containsKey(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO)) {
|
.containsKey(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO)) {
|
||||||
latestApplicationAttemptId =
|
latestApplicationAttemptId = ApplicationAttemptId.fromString(
|
||||||
ConverterUtils
|
eventInfo.get(
|
||||||
.toApplicationAttemptId(
|
|
||||||
eventInfo
|
|
||||||
.get(
|
|
||||||
ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO)
|
ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO)
|
||||||
.toString());
|
.toString());
|
||||||
}
|
}
|
||||||
@ -426,7 +423,7 @@ private static ApplicationReportExt convertToApplicationReport(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
return new ApplicationReportExt(ApplicationReport.newInstance(
|
return new ApplicationReportExt(ApplicationReport.newInstance(
|
||||||
ConverterUtils.toApplicationId(entity.getEntityId()),
|
ApplicationId.fromString(entity.getEntityId()),
|
||||||
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
|
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
|
||||||
diagnosticsInfo, null, createdTime, finishedTime, finalStatus,
|
diagnosticsInfo, null, createdTime, finishedTime, finalStatus,
|
||||||
appResources, null, progress, type, null, appTags, unmanagedApplication,
|
appResources, null, progress, type, null, appTags, unmanagedApplication,
|
||||||
@ -471,7 +468,7 @@ private static ApplicationAttemptReport convertToApplicationAttemptReport(
|
|||||||
if (eventInfo
|
if (eventInfo
|
||||||
.containsKey(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)) {
|
.containsKey(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)) {
|
||||||
amContainerId =
|
amContainerId =
|
||||||
ConverterUtils.toContainerId(eventInfo.get(
|
ContainerId.fromString(eventInfo.get(
|
||||||
AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)
|
AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)
|
||||||
.toString());
|
.toString());
|
||||||
}
|
}
|
||||||
@ -513,7 +510,7 @@ private static ApplicationAttemptReport convertToApplicationAttemptReport(
|
|||||||
if (eventInfo
|
if (eventInfo
|
||||||
.containsKey(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)) {
|
.containsKey(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)) {
|
||||||
amContainerId =
|
amContainerId =
|
||||||
ConverterUtils.toContainerId(eventInfo.get(
|
ContainerId.fromString(eventInfo.get(
|
||||||
AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)
|
AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)
|
||||||
.toString());
|
.toString());
|
||||||
}
|
}
|
||||||
@ -521,7 +518,7 @@ private static ApplicationAttemptReport convertToApplicationAttemptReport(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
return ApplicationAttemptReport.newInstance(
|
return ApplicationAttemptReport.newInstance(
|
||||||
ConverterUtils.toApplicationAttemptId(entity.getEntityId()),
|
ApplicationAttemptId.fromString(entity.getEntityId()),
|
||||||
host, rpcPort, trackingUrl, originalTrackingUrl, diagnosticsInfo,
|
host, rpcPort, trackingUrl, originalTrackingUrl, diagnosticsInfo,
|
||||||
state, amContainerId);
|
state, amContainerId);
|
||||||
}
|
}
|
||||||
@ -610,7 +607,7 @@ private static ContainerReport convertToContainerReport(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
ContainerId containerId =
|
ContainerId containerId =
|
||||||
ConverterUtils.toContainerId(entity.getEntityId());
|
ContainerId.fromString(entity.getEntityId());
|
||||||
String logUrl = null;
|
String logUrl = null;
|
||||||
NodeId allocatedNode = null;
|
NodeId allocatedNode = null;
|
||||||
if (allocatedHost != null) {
|
if (allocatedHost != null) {
|
||||||
@ -623,7 +620,7 @@ private static ContainerReport convertToContainerReport(
|
|||||||
user);
|
user);
|
||||||
}
|
}
|
||||||
return ContainerReport.newInstance(
|
return ContainerReport.newInstance(
|
||||||
ConverterUtils.toContainerId(entity.getEntityId()),
|
ContainerId.fromString(entity.getEntityId()),
|
||||||
Resource.newInstance(allocatedMem, allocatedVcore), allocatedNode,
|
Resource.newInstance(allocatedMem, allocatedVcore), allocatedNode,
|
||||||
Priority.newInstance(allocatedPriority),
|
Priority.newInstance(allocatedPriority),
|
||||||
createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state,
|
createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state,
|
||||||
|
@ -204,7 +204,7 @@ public Map<ApplicationId, ApplicationHistoryData> getAllApplications()
|
|||||||
FileStatus[] files = fs.listStatus(rootDirPath);
|
FileStatus[] files = fs.listStatus(rootDirPath);
|
||||||
for (FileStatus file : files) {
|
for (FileStatus file : files) {
|
||||||
ApplicationId appId =
|
ApplicationId appId =
|
||||||
ConverterUtils.toApplicationId(file.getPath().getName());
|
ApplicationId.fromString(file.getPath().getName());
|
||||||
try {
|
try {
|
||||||
ApplicationHistoryData historyData = getApplication(appId);
|
ApplicationHistoryData historyData = getApplication(appId);
|
||||||
if (historyData != null) {
|
if (historyData != null) {
|
||||||
@ -231,8 +231,8 @@ public Map<ApplicationId, ApplicationHistoryData> getAllApplications()
|
|||||||
HistoryFileReader.Entry entry = hfReader.next();
|
HistoryFileReader.Entry entry = hfReader.next();
|
||||||
if (entry.key.id.startsWith(
|
if (entry.key.id.startsWith(
|
||||||
ConverterUtils.APPLICATION_ATTEMPT_PREFIX)) {
|
ConverterUtils.APPLICATION_ATTEMPT_PREFIX)) {
|
||||||
ApplicationAttemptId appAttemptId =
|
ApplicationAttemptId appAttemptId = ApplicationAttemptId.fromString(
|
||||||
ConverterUtils.toApplicationAttemptId(entry.key.id);
|
entry.key.id);
|
||||||
if (appAttemptId.getApplicationId().equals(appId)) {
|
if (appAttemptId.getApplicationId().equals(appId)) {
|
||||||
ApplicationAttemptHistoryData historyData =
|
ApplicationAttemptHistoryData historyData =
|
||||||
historyDataMap.get(appAttemptId);
|
historyDataMap.get(appAttemptId);
|
||||||
@ -385,7 +385,7 @@ public Map<ContainerId, ContainerHistoryData> getContainers(
|
|||||||
HistoryFileReader.Entry entry = hfReader.next();
|
HistoryFileReader.Entry entry = hfReader.next();
|
||||||
if (entry.key.id.startsWith(ConverterUtils.CONTAINER_PREFIX)) {
|
if (entry.key.id.startsWith(ConverterUtils.CONTAINER_PREFIX)) {
|
||||||
ContainerId containerId =
|
ContainerId containerId =
|
||||||
ConverterUtils.toContainerId(entry.key.id);
|
ContainerId.fromString(entry.key.id);
|
||||||
if (containerId.getApplicationAttemptId().equals(appAttemptId)) {
|
if (containerId.getApplicationAttemptId().equals(appAttemptId)) {
|
||||||
ContainerHistoryData historyData =
|
ContainerHistoryData historyData =
|
||||||
historyDataMap.get(containerId);
|
historyDataMap.get(containerId);
|
||||||
|
@ -112,7 +112,7 @@ public static LocalResource newLocalResource(URL url, LocalResourceType type,
|
|||||||
public static LocalResource newLocalResource(URI uri,
|
public static LocalResource newLocalResource(URI uri,
|
||||||
LocalResourceType type, LocalResourceVisibility visibility, long size,
|
LocalResourceType type, LocalResourceVisibility visibility, long size,
|
||||||
long timestamp, boolean shouldBeUploadedToSharedCache) {
|
long timestamp, boolean shouldBeUploadedToSharedCache) {
|
||||||
return newLocalResource(ConverterUtils.getYarnUrlFromURI(uri), type,
|
return newLocalResource(URL.fromURI(uri), type,
|
||||||
visibility, size, timestamp, shouldBeUploadedToSharedCache);
|
visibility, size, timestamp, shouldBeUploadedToSharedCache);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -65,7 +65,7 @@ protected void render(Block html) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
appAttemptId = ConverterUtils.toApplicationAttemptId(attemptid);
|
appAttemptId = ApplicationAttemptId.fromString(attemptid);
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
puts("Invalid application attempt ID: " + attemptid);
|
puts("Invalid application attempt ID: " + attemptid);
|
||||||
return;
|
return;
|
||||||
|
@ -59,7 +59,7 @@ protected void render(Block html) {
|
|||||||
|
|
||||||
ContainerId containerId = null;
|
ContainerId containerId = null;
|
||||||
try {
|
try {
|
||||||
containerId = ConverterUtils.toContainerId(containerid);
|
containerId = ContainerId.fromString(containerid);
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
puts("Invalid container ID: " + containerid);
|
puts("Invalid container ID: " + containerid);
|
||||||
return;
|
return;
|
||||||
|
@ -431,7 +431,7 @@ protected static ApplicationId parseApplicationId(String appId) {
|
|||||||
}
|
}
|
||||||
ApplicationId aid = null;
|
ApplicationId aid = null;
|
||||||
try {
|
try {
|
||||||
aid = ConverterUtils.toApplicationId(appId);
|
aid = ApplicationId.fromString(appId);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new BadRequestException(e);
|
throw new BadRequestException(e);
|
||||||
}
|
}
|
||||||
@ -449,7 +449,7 @@ protected static ApplicationAttemptId parseApplicationAttemptId(
|
|||||||
}
|
}
|
||||||
ApplicationAttemptId aaid = null;
|
ApplicationAttemptId aaid = null;
|
||||||
try {
|
try {
|
||||||
aaid = ConverterUtils.toApplicationAttemptId(appAttemptId);
|
aaid = ApplicationAttemptId.fromString(appAttemptId);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new BadRequestException(e);
|
throw new BadRequestException(e);
|
||||||
}
|
}
|
||||||
@ -466,7 +466,7 @@ protected static ContainerId parseContainerId(String containerId) {
|
|||||||
}
|
}
|
||||||
ContainerId cid = null;
|
ContainerId cid = null;
|
||||||
try {
|
try {
|
||||||
cid = ConverterUtils.toContainerId(containerId);
|
cid = ContainerId.fromString(containerId);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new BadRequestException(e);
|
throw new BadRequestException(e);
|
||||||
}
|
}
|
||||||
|
@ -165,11 +165,10 @@ public int launchContainer(ContainerStartContext ctx) throws IOException {
|
|||||||
ContainerId containerId = container.getContainerId();
|
ContainerId containerId = container.getContainerId();
|
||||||
|
|
||||||
// create container dirs on all disks
|
// create container dirs on all disks
|
||||||
String containerIdStr = ConverterUtils.toString(containerId);
|
String containerIdStr = containerId.toString();
|
||||||
String appIdStr =
|
String appIdStr =
|
||||||
ConverterUtils.toString(
|
|
||||||
containerId.getApplicationAttemptId().
|
containerId.getApplicationAttemptId().
|
||||||
getApplicationId());
|
getApplicationId().toString();
|
||||||
for (String sLocalDir : localDirs) {
|
for (String sLocalDir : localDirs) {
|
||||||
Path usersdir = new Path(sLocalDir, ContainerLocalizer.USERCACHE);
|
Path usersdir = new Path(sLocalDir, ContainerLocalizer.USERCACHE);
|
||||||
Path userdir = new Path(usersdir, user);
|
Path userdir = new Path(usersdir, user);
|
||||||
|
@ -195,9 +195,9 @@ public int launchContainer(ContainerStartContext ctx) throws IOException {
|
|||||||
ContainerId containerId = container.getContainerId();
|
ContainerId containerId = container.getContainerId();
|
||||||
|
|
||||||
// create container dirs on all disks
|
// create container dirs on all disks
|
||||||
String containerIdStr = ConverterUtils.toString(containerId);
|
String containerIdStr = containerId.toString();
|
||||||
String appIdStr = ConverterUtils.toString(
|
String appIdStr =
|
||||||
containerId.getApplicationAttemptId().getApplicationId());
|
containerId.getApplicationAttemptId().getApplicationId().toString();
|
||||||
for (String sLocalDir : localDirs) {
|
for (String sLocalDir : localDirs) {
|
||||||
Path usersdir = new Path(sLocalDir, ContainerLocalizer.USERCACHE);
|
Path usersdir = new Path(sLocalDir, ContainerLocalizer.USERCACHE);
|
||||||
Path userdir = new Path(usersdir, userName);
|
Path userdir = new Path(usersdir, userName);
|
||||||
|
@ -297,7 +297,7 @@ public int launchContainer(ContainerStartContext ctx) throws IOException {
|
|||||||
String runAsUser = getRunAsUser(user);
|
String runAsUser = getRunAsUser(user);
|
||||||
|
|
||||||
ContainerId containerId = container.getContainerId();
|
ContainerId containerId = container.getContainerId();
|
||||||
String containerIdStr = ConverterUtils.toString(containerId);
|
String containerIdStr = containerId.toString();
|
||||||
|
|
||||||
resourcesHandler.preExecute(containerId,
|
resourcesHandler.preExecute(containerId,
|
||||||
container.getResource());
|
container.getResource());
|
||||||
|
@ -1164,7 +1164,7 @@ public void handle(ContainerEvent event) {
|
|||||||
public String toString() {
|
public String toString() {
|
||||||
this.readLock.lock();
|
this.readLock.lock();
|
||||||
try {
|
try {
|
||||||
return ConverterUtils.toString(this.containerId);
|
return this.containerId.toString();
|
||||||
} finally {
|
} finally {
|
||||||
this.readLock.unlock();
|
this.readLock.unlock();
|
||||||
}
|
}
|
||||||
|
@ -155,7 +155,7 @@ public Integer call() {
|
|||||||
final ContainerLaunchContext launchContext = container.getLaunchContext();
|
final ContainerLaunchContext launchContext = container.getLaunchContext();
|
||||||
Map<Path,List<String>> localResources = null;
|
Map<Path,List<String>> localResources = null;
|
||||||
ContainerId containerID = container.getContainerId();
|
ContainerId containerID = container.getContainerId();
|
||||||
String containerIdStr = ConverterUtils.toString(containerID);
|
String containerIdStr = containerID.toString();
|
||||||
final List<String> command = launchContext.getCommands();
|
final List<String> command = launchContext.getCommands();
|
||||||
int ret = -1;
|
int ret = -1;
|
||||||
|
|
||||||
@ -377,7 +377,7 @@ protected String getPidFileSubpath(String appIdStr, String containerIdStr) {
|
|||||||
@SuppressWarnings("unchecked") // dispatcher not typed
|
@SuppressWarnings("unchecked") // dispatcher not typed
|
||||||
public void cleanupContainer() throws IOException {
|
public void cleanupContainer() throws IOException {
|
||||||
ContainerId containerId = container.getContainerId();
|
ContainerId containerId = container.getContainerId();
|
||||||
String containerIdStr = ConverterUtils.toString(containerId);
|
String containerIdStr = containerId.toString();
|
||||||
LOG.info("Cleaning up container " + containerIdStr);
|
LOG.info("Cleaning up container " + containerIdStr);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@ -473,7 +473,7 @@ public void signalContainer(SignalContainerCommand command)
|
|||||||
throws IOException {
|
throws IOException {
|
||||||
ContainerId containerId =
|
ContainerId containerId =
|
||||||
container.getContainerTokenIdentifier().getContainerID();
|
container.getContainerTokenIdentifier().getContainerID();
|
||||||
String containerIdStr = ConverterUtils.toString(containerId);
|
String containerIdStr = containerId.toString();
|
||||||
String user = container.getUser();
|
String user = container.getUser();
|
||||||
Signal signal = translateCommandToSignal(command);
|
Signal signal = translateCommandToSignal(command);
|
||||||
if (signal.equals(Signal.NULL)) {
|
if (signal.equals(Signal.NULL)) {
|
||||||
@ -565,7 +565,7 @@ public static Signal translateCommandToSignal(
|
|||||||
*/
|
*/
|
||||||
private String getContainerPid(Path pidFilePath) throws Exception {
|
private String getContainerPid(Path pidFilePath) throws Exception {
|
||||||
String containerIdStr =
|
String containerIdStr =
|
||||||
ConverterUtils.toString(container.getContainerId());
|
container.getContainerId().toString();
|
||||||
String processId = null;
|
String processId = null;
|
||||||
LOG.debug("Accessing pid for container " + containerIdStr
|
LOG.debug("Accessing pid for container " + containerIdStr
|
||||||
+ " from pid file " + pidFilePath);
|
+ " from pid file " + pidFilePath);
|
||||||
|
@ -68,9 +68,9 @@ public RecoveredContainerLaunch(Context context, Configuration configuration,
|
|||||||
public Integer call() {
|
public Integer call() {
|
||||||
int retCode = ExitCode.LOST.getExitCode();
|
int retCode = ExitCode.LOST.getExitCode();
|
||||||
ContainerId containerId = container.getContainerId();
|
ContainerId containerId = container.getContainerId();
|
||||||
String appIdStr = ConverterUtils.toString(
|
String appIdStr =
|
||||||
containerId.getApplicationAttemptId().getApplicationId());
|
containerId.getApplicationAttemptId().getApplicationId().toString();
|
||||||
String containerIdStr = ConverterUtils.toString(containerId);
|
String containerIdStr = containerId.toString();
|
||||||
|
|
||||||
dispatcher.getEventHandler().handle(new ContainerEvent(containerId,
|
dispatcher.getEventHandler().handle(new ContainerEvent(containerId,
|
||||||
ContainerEventType.CONTAINER_LAUNCHED));
|
ContainerEventType.CONTAINER_LAUNCHED));
|
||||||
|
@ -56,6 +56,7 @@
|
|||||||
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
|
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResource;
|
import org.apache.hadoop.yarn.api.records.LocalResource;
|
||||||
import org.apache.hadoop.yarn.api.records.SerializedException;
|
import org.apache.hadoop.yarn.api.records.SerializedException;
|
||||||
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
@ -295,7 +296,7 @@ private LocalizerStatus createStatus() throws InterruptedException {
|
|||||||
try {
|
try {
|
||||||
Path localPath = fPath.get();
|
Path localPath = fPath.get();
|
||||||
stat.setLocalPath(
|
stat.setLocalPath(
|
||||||
ConverterUtils.getYarnUrlFromPath(localPath));
|
URL.fromPath(localPath));
|
||||||
stat.setLocalSize(
|
stat.setLocalSize(
|
||||||
FileUtil.getDU(new File(localPath.getParent().toUri())));
|
FileUtil.getDU(new File(localPath.getParent().toUri())));
|
||||||
stat.setStatus(ResourceStatusType.FETCH_SUCCESS);
|
stat.setStatus(ResourceStatusType.FETCH_SUCCESS);
|
||||||
|
@ -43,7 +43,7 @@ public class LocalResourceRequest
|
|||||||
*/
|
*/
|
||||||
public LocalResourceRequest(LocalResource resource)
|
public LocalResourceRequest(LocalResource resource)
|
||||||
throws URISyntaxException {
|
throws URISyntaxException {
|
||||||
this(ConverterUtils.getPathFromYarnURL(resource.getResource()),
|
this(resource.getResource().toPath(),
|
||||||
resource.getTimestamp(),
|
resource.getTimestamp(),
|
||||||
resource.getType(),
|
resource.getType(),
|
||||||
resource.getVisibility(),
|
resource.getVisibility(),
|
||||||
@ -133,7 +133,7 @@ public LocalResourceType getType() {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public URL getResource() {
|
public URL getResource() {
|
||||||
return ConverterUtils.getYarnUrlFromPath(loc);
|
return URL.fromPath(loc);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -79,6 +79,7 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResource;
|
import org.apache.hadoop.yarn.api.records.LocalResource;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
||||||
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.event.Dispatcher;
|
import org.apache.hadoop.yarn.event.Dispatcher;
|
||||||
@ -301,7 +302,7 @@ public void recoverLocalizedResources(RecoveredLocalizationState state)
|
|||||||
trackerState = appEntry.getValue();
|
trackerState = appEntry.getValue();
|
||||||
if (!trackerState.isEmpty()) {
|
if (!trackerState.isEmpty()) {
|
||||||
ApplicationId appId = appEntry.getKey();
|
ApplicationId appId = appEntry.getKey();
|
||||||
String appIdStr = ConverterUtils.toString(appId);
|
String appIdStr = appId.toString();
|
||||||
LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
|
LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
|
||||||
appId, dispatcher, false, super.getConfig(), stateStore);
|
appId, dispatcher, false, super.getConfig(), stateStore);
|
||||||
LocalResourcesTracker oldTracker = appRsrc.putIfAbsent(appIdStr,
|
LocalResourcesTracker oldTracker = appRsrc.putIfAbsent(appIdStr,
|
||||||
@ -442,7 +443,7 @@ private void handleInitApplicationResources(Application app) {
|
|||||||
String userName = app.getUser();
|
String userName = app.getUser();
|
||||||
privateRsrc.putIfAbsent(userName, new LocalResourcesTrackerImpl(userName,
|
privateRsrc.putIfAbsent(userName, new LocalResourcesTrackerImpl(userName,
|
||||||
null, dispatcher, true, super.getConfig(), stateStore));
|
null, dispatcher, true, super.getConfig(), stateStore));
|
||||||
String appIdStr = ConverterUtils.toString(app.getAppId());
|
String appIdStr = app.getAppId().toString();
|
||||||
appRsrc.putIfAbsent(appIdStr, new LocalResourcesTrackerImpl(app.getUser(),
|
appRsrc.putIfAbsent(appIdStr, new LocalResourcesTrackerImpl(app.getUser(),
|
||||||
app.getAppId(), dispatcher, false, super.getConfig(), stateStore));
|
app.getAppId(), dispatcher, false, super.getConfig(), stateStore));
|
||||||
// 1) Signal container init
|
// 1) Signal container init
|
||||||
@ -491,7 +492,7 @@ private void handleInitContainerResources(
|
|||||||
private void handleContainerResourcesLocalized(
|
private void handleContainerResourcesLocalized(
|
||||||
ContainerLocalizationEvent event) {
|
ContainerLocalizationEvent event) {
|
||||||
Container c = event.getContainer();
|
Container c = event.getContainer();
|
||||||
String locId = ConverterUtils.toString(c.getContainerId());
|
String locId = c.getContainerId().toString();
|
||||||
localizerTracker.endContainerLocalization(locId);
|
localizerTracker.endContainerLocalization(locId);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -528,14 +529,15 @@ private void handleCleanupContainerResources(
|
|||||||
c.getContainerId()));
|
c.getContainerId()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
String locId = ConverterUtils.toString(c.getContainerId());
|
String locId = c.getContainerId().toString();
|
||||||
localizerTracker.cleanupPrivLocalizers(locId);
|
localizerTracker.cleanupPrivLocalizers(locId);
|
||||||
|
|
||||||
// Delete the container directories
|
// Delete the container directories
|
||||||
String userName = c.getUser();
|
String userName = c.getUser();
|
||||||
String containerIDStr = c.toString();
|
String containerIDStr = c.toString();
|
||||||
String appIDStr = ConverterUtils.toString(
|
String appIDStr =
|
||||||
c.getContainerId().getApplicationAttemptId().getApplicationId());
|
c.getContainerId().getApplicationAttemptId().getApplicationId()
|
||||||
|
.toString();
|
||||||
|
|
||||||
// Try deleting from good local dirs and full local dirs because a dir might
|
// Try deleting from good local dirs and full local dirs because a dir might
|
||||||
// have gone bad while the app was running(disk full). In addition
|
// have gone bad while the app was running(disk full). In addition
|
||||||
@ -583,7 +585,7 @@ private void handleDestroyApplicationResources(Application application) {
|
|||||||
ApplicationId appId = application.getAppId();
|
ApplicationId appId = application.getAppId();
|
||||||
String appIDStr = application.toString();
|
String appIDStr = application.toString();
|
||||||
LocalResourcesTracker appLocalRsrcsTracker =
|
LocalResourcesTracker appLocalRsrcsTracker =
|
||||||
appRsrc.remove(ConverterUtils.toString(appId));
|
appRsrc.remove(appId.toString());
|
||||||
if (appLocalRsrcsTracker != null) {
|
if (appLocalRsrcsTracker != null) {
|
||||||
for (LocalizedResource rsrc : appLocalRsrcsTracker ) {
|
for (LocalizedResource rsrc : appLocalRsrcsTracker ) {
|
||||||
Path localPath = rsrc.getLocalPath();
|
Path localPath = rsrc.getLocalPath();
|
||||||
@ -637,7 +639,7 @@ LocalResourcesTracker getLocalResourcesTracker(
|
|||||||
case PRIVATE:
|
case PRIVATE:
|
||||||
return privateRsrc.get(user);
|
return privateRsrc.get(user);
|
||||||
case APPLICATION:
|
case APPLICATION:
|
||||||
return appRsrc.get(ConverterUtils.toString(appId));
|
return appRsrc.get(appId.toString());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -977,7 +979,7 @@ private LocalResource findNextResource() {
|
|||||||
LocalResourceRequest nextRsrc = nRsrc.getRequest();
|
LocalResourceRequest nextRsrc = nRsrc.getRequest();
|
||||||
LocalResource next =
|
LocalResource next =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
next.setResource(ConverterUtils.getYarnUrlFromPath(nextRsrc
|
next.setResource(URL.fromPath(nextRsrc
|
||||||
.getPath()));
|
.getPath()));
|
||||||
next.setTimestamp(nextRsrc.getTimestamp());
|
next.setTimestamp(nextRsrc.getTimestamp());
|
||||||
next.setType(nextRsrc.getType());
|
next.setType(nextRsrc.getType());
|
||||||
@ -1028,8 +1030,8 @@ LocalizerHeartbeatResponse processHeartbeat(
|
|||||||
try {
|
try {
|
||||||
getLocalResourcesTracker(req.getVisibility(), user, applicationId)
|
getLocalResourcesTracker(req.getVisibility(), user, applicationId)
|
||||||
.handle(
|
.handle(
|
||||||
new ResourceLocalizedEvent(req, ConverterUtils
|
new ResourceLocalizedEvent(req, stat.getLocalPath().toPath(),
|
||||||
.getPathFromYarnURL(stat.getLocalPath()), stat.getLocalSize()));
|
stat.getLocalSize()));
|
||||||
} catch (URISyntaxException e) { }
|
} catch (URISyntaxException e) { }
|
||||||
|
|
||||||
// unlocking the resource and removing it from scheduled resource
|
// unlocking the resource and removing it from scheduled resource
|
||||||
@ -1142,8 +1144,8 @@ public void run() {
|
|||||||
.setNmPrivateContainerTokens(nmPrivateCTokensPath)
|
.setNmPrivateContainerTokens(nmPrivateCTokensPath)
|
||||||
.setNmAddr(localizationServerAddress)
|
.setNmAddr(localizationServerAddress)
|
||||||
.setUser(context.getUser())
|
.setUser(context.getUser())
|
||||||
.setAppId(ConverterUtils.toString(context.getContainerId()
|
.setAppId(context.getContainerId()
|
||||||
.getApplicationAttemptId().getApplicationId()))
|
.getApplicationAttemptId().getApplicationId().toString())
|
||||||
.setLocId(localizerId)
|
.setLocId(localizerId)
|
||||||
.setDirsHandler(dirsHandler)
|
.setDirsHandler(dirsHandler)
|
||||||
.build());
|
.build());
|
||||||
|
@ -37,7 +37,7 @@ public class LocalizerResourceRequestEvent extends LocalizerEvent {
|
|||||||
public LocalizerResourceRequestEvent(LocalizedResource resource,
|
public LocalizerResourceRequestEvent(LocalizedResource resource,
|
||||||
LocalResourceVisibility vis, LocalizerContext context, String pattern) {
|
LocalResourceVisibility vis, LocalizerContext context, String pattern) {
|
||||||
super(LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION,
|
super(LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION,
|
||||||
ConverterUtils.toString(context.getContainerId()));
|
context.getContainerId().toString());
|
||||||
this.vis = vis;
|
this.vis = vis;
|
||||||
this.context = context;
|
this.context = context;
|
||||||
this.resource = resource;
|
this.resource = resource;
|
||||||
|
@ -211,7 +211,7 @@ boolean verifyAccess() throws IOException {
|
|||||||
|
|
||||||
final Path remotePath;
|
final Path remotePath;
|
||||||
try {
|
try {
|
||||||
remotePath = ConverterUtils.getPathFromYarnURL(resource.getResource());
|
remotePath = resource.getResource().toPath();
|
||||||
} catch (URISyntaxException e) {
|
} catch (URISyntaxException e) {
|
||||||
throw new IOException("Invalid resource", e);
|
throw new IOException("Invalid resource", e);
|
||||||
}
|
}
|
||||||
|
@ -147,7 +147,7 @@ public AppLogAggregatorImpl(Dispatcher dispatcher,
|
|||||||
this.conf = conf;
|
this.conf = conf;
|
||||||
this.delService = deletionService;
|
this.delService = deletionService;
|
||||||
this.appId = appId;
|
this.appId = appId;
|
||||||
this.applicationId = ConverterUtils.toString(appId);
|
this.applicationId = appId.toString();
|
||||||
this.userUgi = userUgi;
|
this.userUgi = userUgi;
|
||||||
this.dirsHandler = dirsHandler;
|
this.dirsHandler = dirsHandler;
|
||||||
this.remoteNodeLogFileForApp = remoteNodeLogFileForApp;
|
this.remoteNodeLogFileForApp = remoteNodeLogFileForApp;
|
||||||
|
@ -181,7 +181,7 @@ public List<RecoveredContainerState> loadContainersState()
|
|||||||
if (idEndPos < 0) {
|
if (idEndPos < 0) {
|
||||||
throw new IOException("Unable to determine container in key: " + key);
|
throw new IOException("Unable to determine container in key: " + key);
|
||||||
}
|
}
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(
|
ContainerId containerId = ContainerId.fromString(
|
||||||
key.substring(CONTAINERS_KEY_PREFIX.length(), idEndPos));
|
key.substring(CONTAINERS_KEY_PREFIX.length(), idEndPos));
|
||||||
String keyPrefix = key.substring(0, idEndPos+1);
|
String keyPrefix = key.substring(0, idEndPos+1);
|
||||||
RecoveredContainerState rcs = loadContainerState(containerId,
|
RecoveredContainerState rcs = loadContainerState(containerId,
|
||||||
@ -545,7 +545,7 @@ private RecoveredUserResources loadUserLocalizedResources(
|
|||||||
throw new IOException("Unable to determine appID in resource key: "
|
throw new IOException("Unable to determine appID in resource key: "
|
||||||
+ key);
|
+ key);
|
||||||
}
|
}
|
||||||
ApplicationId appId = ConverterUtils.toApplicationId(
|
ApplicationId appId = ApplicationId.fromString(
|
||||||
key.substring(appIdStartPos, appIdEndPos));
|
key.substring(appIdStartPos, appIdEndPos));
|
||||||
userResources.appTrackerStates.put(appId,
|
userResources.appTrackerStates.put(appId,
|
||||||
loadResourceTrackerState(iter, key.substring(0, appIdEndPos+1)));
|
loadResourceTrackerState(iter, key.substring(0, appIdEndPos+1)));
|
||||||
@ -713,7 +713,7 @@ public RecoveredNMTokensState loadNMTokensState() throws IOException {
|
|||||||
ApplicationAttemptId.appAttemptIdStrPrefix)) {
|
ApplicationAttemptId.appAttemptIdStrPrefix)) {
|
||||||
ApplicationAttemptId attempt;
|
ApplicationAttemptId attempt;
|
||||||
try {
|
try {
|
||||||
attempt = ConverterUtils.toApplicationAttemptId(key);
|
attempt = ApplicationAttemptId.fromString(key);
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
throw new IOException("Bad application master key state for "
|
throw new IOException("Bad application master key state for "
|
||||||
+ fullKey, e);
|
+ fullKey, e);
|
||||||
@ -817,7 +817,7 @@ private static void loadContainerToken(RecoveredContainerTokensState state,
|
|||||||
ContainerId containerId;
|
ContainerId containerId;
|
||||||
Long expTime;
|
Long expTime;
|
||||||
try {
|
try {
|
||||||
containerId = ConverterUtils.toContainerId(containerIdStr);
|
containerId = ContainerId.fromString(containerIdStr);
|
||||||
expTime = Long.parseLong(asString(value));
|
expTime = Long.parseLong(asString(value));
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
throw new IOException("Bad container token state for " + key, e);
|
throw new IOException("Bad container token state for " + key, e);
|
||||||
@ -879,7 +879,7 @@ public RecoveredLogDeleterState loadLogDeleterState() throws IOException {
|
|||||||
String appIdStr = fullKey.substring(logDeleterKeyPrefixLength);
|
String appIdStr = fullKey.substring(logDeleterKeyPrefixLength);
|
||||||
ApplicationId appId = null;
|
ApplicationId appId = null;
|
||||||
try {
|
try {
|
||||||
appId = ConverterUtils.toApplicationId(appIdStr);
|
appId = ApplicationId.fromString(appIdStr);
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
LOG.warn("Skipping unknown log deleter key " + fullKey);
|
LOG.warn("Skipping unknown log deleter key " + fullKey);
|
||||||
continue;
|
continue;
|
||||||
|
@ -28,7 +28,7 @@ public class NodeManagerBuilderUtils {
|
|||||||
|
|
||||||
public static ResourceLocalizationSpec newResourceLocalizationSpec(
|
public static ResourceLocalizationSpec newResourceLocalizationSpec(
|
||||||
LocalResource rsrc, Path path) {
|
LocalResource rsrc, Path path) {
|
||||||
URL local = ConverterUtils.getYarnUrlFromPath(path);
|
URL local = URL.fromPath(path);
|
||||||
ResourceLocalizationSpec resourceLocalizationSpec =
|
ResourceLocalizationSpec resourceLocalizationSpec =
|
||||||
Records.newRecord(ResourceLocalizationSpec.class);
|
Records.newRecord(ResourceLocalizationSpec.class);
|
||||||
resourceLocalizationSpec.setDestinationDirectory(local);
|
resourceLocalizationSpec.setDestinationDirectory(local);
|
||||||
|
@ -27,6 +27,7 @@
|
|||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.util.Shell;
|
import org.apache.hadoop.util.Shell;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -69,7 +70,7 @@ public static String getProcessId(Path path) throws IOException {
|
|||||||
// On Windows, pid is expected to be a container ID, so find first
|
// On Windows, pid is expected to be a container ID, so find first
|
||||||
// line that parses successfully as a container ID.
|
// line that parses successfully as a container ID.
|
||||||
try {
|
try {
|
||||||
ConverterUtils.toContainerId(temp);
|
ContainerId.fromString(temp);
|
||||||
processId = temp;
|
processId = temp;
|
||||||
break;
|
break;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
@ -75,9 +75,7 @@ public ApplicationBlock(Context nmContext, Configuration conf) {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void render(Block html) {
|
protected void render(Block html) {
|
||||||
ApplicationId applicationID =
|
ApplicationId applicationID = ApplicationId.fromString($(APPLICATION_ID));
|
||||||
ConverterUtils.toApplicationId(this.recordFactory,
|
|
||||||
$(APPLICATION_ID));
|
|
||||||
Application app = this.nmContext.getApplications().get(applicationID);
|
Application app = this.nmContext.getApplications().get(applicationID);
|
||||||
AppInfo info = new AppInfo(app);
|
AppInfo info = new AppInfo(app);
|
||||||
info("Application's information")
|
info("Application's information")
|
||||||
|
@ -92,7 +92,7 @@ protected void render(Block html) {
|
|||||||
|
|
||||||
ContainerId containerId;
|
ContainerId containerId;
|
||||||
try {
|
try {
|
||||||
containerId = ConverterUtils.toContainerId($(CONTAINER_ID));
|
containerId = ContainerId.fromString($(CONTAINER_ID));
|
||||||
} catch (IllegalArgumentException ex) {
|
} catch (IllegalArgumentException ex) {
|
||||||
html.h1("Invalid container ID: " + $(CONTAINER_ID));
|
html.h1("Invalid container ID: " + $(CONTAINER_ID));
|
||||||
return;
|
return;
|
||||||
|
@ -78,8 +78,8 @@ static List<File> getContainerLogDirs(ContainerId containerId,
|
|||||||
List<File> containerLogDirs = new ArrayList<File>(logDirs.size());
|
List<File> containerLogDirs = new ArrayList<File>(logDirs.size());
|
||||||
for (String logDir : logDirs) {
|
for (String logDir : logDirs) {
|
||||||
logDir = new File(logDir).toURI().getPath();
|
logDir = new File(logDir).toURI().getPath();
|
||||||
String appIdStr = ConverterUtils.toString(containerId
|
String appIdStr = containerId
|
||||||
.getApplicationAttemptId().getApplicationId());
|
.getApplicationAttemptId().getApplicationId().toString();
|
||||||
File appLogDir = new File(logDir, appIdStr);
|
File appLogDir = new File(logDir, appIdStr);
|
||||||
containerLogDirs.add(new File(appLogDir, containerId.toString()));
|
containerLogDirs.add(new File(appLogDir, containerId.toString()));
|
||||||
}
|
}
|
||||||
@ -160,7 +160,7 @@ private static void checkState(ContainerState state) {
|
|||||||
|
|
||||||
public static FileInputStream openLogFileForRead(String containerIdStr, File logFile,
|
public static FileInputStream openLogFileForRead(String containerIdStr, File logFile,
|
||||||
Context context) throws IOException {
|
Context context) throws IOException {
|
||||||
ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
|
ContainerId containerId = ContainerId.fromString(containerIdStr);
|
||||||
ApplicationId applicationId = containerId.getApplicationAttemptId()
|
ApplicationId applicationId = containerId.getApplicationAttemptId()
|
||||||
.getApplicationId();
|
.getApplicationId();
|
||||||
String user = context.getApplications().get(
|
String user = context.getApplications().get(
|
||||||
|
@ -63,7 +63,7 @@ public ContainerBlock(Context nmContext) {
|
|||||||
protected void render(Block html) {
|
protected void render(Block html) {
|
||||||
ContainerId containerID;
|
ContainerId containerID;
|
||||||
try {
|
try {
|
||||||
containerID = ConverterUtils.toContainerId($(CONTAINER_ID));
|
containerID = ContainerId.fromString($(CONTAINER_ID));
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
html.p()._("Invalid containerId " + $(CONTAINER_ID))._();
|
html.p()._("Invalid containerId " + $(CONTAINER_ID))._();
|
||||||
return;
|
return;
|
||||||
|
@ -181,7 +181,7 @@ public ContainerInfo getNodeContainer(@javax.ws.rs.core.Context
|
|||||||
ContainerId containerId = null;
|
ContainerId containerId = null;
|
||||||
init();
|
init();
|
||||||
try {
|
try {
|
||||||
containerId = ConverterUtils.toContainerId(id);
|
containerId = ContainerId.fromString(id);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new BadRequestException("invalid container id, " + id);
|
throw new BadRequestException("invalid container id, " + id);
|
||||||
}
|
}
|
||||||
@ -218,7 +218,7 @@ public Response getLogs(@PathParam("containerid") String containerIdStr,
|
|||||||
@PathParam("filename") String filename) {
|
@PathParam("filename") String filename) {
|
||||||
ContainerId containerId;
|
ContainerId containerId;
|
||||||
try {
|
try {
|
||||||
containerId = ConverterUtils.toContainerId(containerIdStr);
|
containerId = ContainerId.fromString(containerIdStr);
|
||||||
} catch (IllegalArgumentException ex) {
|
} catch (IllegalArgumentException ex) {
|
||||||
return Response.status(Status.BAD_REQUEST).build();
|
return Response.status(Status.BAD_REQUEST).build();
|
||||||
}
|
}
|
||||||
|
@ -42,14 +42,14 @@ public AppInfo() {
|
|||||||
} // JAXB needs this
|
} // JAXB needs this
|
||||||
|
|
||||||
public AppInfo(final Application app) {
|
public AppInfo(final Application app) {
|
||||||
this.id = ConverterUtils.toString(app.getAppId());
|
this.id = app.getAppId().toString();
|
||||||
this.state = app.getApplicationState().toString();
|
this.state = app.getApplicationState().toString();
|
||||||
this.user = app.getUser();
|
this.user = app.getUser();
|
||||||
|
|
||||||
this.containerids = new ArrayList<String>();
|
this.containerids = new ArrayList<String>();
|
||||||
Map<ContainerId, Container> appContainers = app.getContainers();
|
Map<ContainerId, Container> appContainers = app.getContainers();
|
||||||
for (ContainerId containerId : appContainers.keySet()) {
|
for (ContainerId containerId : appContainers.keySet()) {
|
||||||
String containerIdStr = ConverterUtils.toString(containerId);
|
String containerIdStr = containerId.toString();
|
||||||
containerids.add(containerIdStr);
|
containerids.add(containerIdStr);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -118,7 +118,7 @@ public void testClearLocalDirWhenNodeReboot() throws IOException,
|
|||||||
ContainerId cId = createContainerId();
|
ContainerId cId = createContainerId();
|
||||||
|
|
||||||
URL localResourceUri =
|
URL localResourceUri =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS.makeQualified(new Path(
|
URL.fromPath(localFS.makeQualified(new Path(
|
||||||
localResourceDir.getAbsolutePath())));
|
localResourceDir.getAbsolutePath())));
|
||||||
|
|
||||||
LocalResource localResource =
|
LocalResource localResource =
|
||||||
|
@ -741,7 +741,7 @@ public void startContainer()
|
|||||||
ContainerLaunchContext containerLaunchContext =
|
ContainerLaunchContext containerLaunchContext =
|
||||||
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
|
@ -200,7 +200,7 @@ public static void startContainer(NodeManager nm, ContainerId cId,
|
|||||||
.getCanonicalHostName(), port);
|
.getCanonicalHostName(), port);
|
||||||
|
|
||||||
URL localResourceUri =
|
URL localResourceUri =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource localResource =
|
LocalResource localResource =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
|
@ -25,6 +25,7 @@
|
|||||||
import java.net.URISyntaxException;
|
import java.net.URISyntaxException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
|
||||||
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
@ -62,8 +63,7 @@ static RecordFactory createPBRecordFactory() {
|
|||||||
static LocalResource createResource() {
|
static LocalResource createResource() {
|
||||||
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
assertTrue(ret instanceof LocalResourcePBImpl);
|
assertTrue(ret instanceof LocalResourcePBImpl);
|
||||||
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(
|
ret.setResource(URL.fromPath(new Path("hdfs://y.ak:8020/foo/bar")));
|
||||||
"hdfs://y.ak:8020/foo/bar")));
|
|
||||||
ret.setSize(4344L);
|
ret.setSize(4344L);
|
||||||
ret.setTimestamp(3141592653589793L);
|
ret.setTimestamp(3141592653589793L);
|
||||||
ret.setVisibility(LocalResourceVisibility.PUBLIC);
|
ret.setVisibility(LocalResourceVisibility.PUBLIC);
|
||||||
@ -76,7 +76,7 @@ static LocalResourceStatus createLocalResourceStatus() {
|
|||||||
assertTrue(ret instanceof LocalResourceStatusPBImpl);
|
assertTrue(ret instanceof LocalResourceStatusPBImpl);
|
||||||
ret.setResource(createResource());
|
ret.setResource(createResource());
|
||||||
ret.setLocalPath(
|
ret.setLocalPath(
|
||||||
ConverterUtils.getYarnUrlFromPath(
|
URL.fromPath(
|
||||||
new Path("file:///local/foo/bar")));
|
new Path("file:///local/foo/bar")));
|
||||||
ret.setStatus(ResourceStatusType.FETCH_SUCCESS);
|
ret.setStatus(ResourceStatusType.FETCH_SUCCESS);
|
||||||
ret.setLocalSize(4443L);
|
ret.setLocalSize(4443L);
|
||||||
@ -109,8 +109,8 @@ static LocalizerHeartbeatResponse createLocalizerHeartbeatResponse()
|
|||||||
ResourceLocalizationSpec resource =
|
ResourceLocalizationSpec resource =
|
||||||
recordFactory.newRecordInstance(ResourceLocalizationSpec.class);
|
recordFactory.newRecordInstance(ResourceLocalizationSpec.class);
|
||||||
resource.setResource(rsrc);
|
resource.setResource(rsrc);
|
||||||
resource.setDestinationDirectory(ConverterUtils
|
resource.setDestinationDirectory(
|
||||||
.getYarnUrlFromPath(new Path("/tmp" + System.currentTimeMillis())));
|
URL.fromPath((new Path("/tmp" + System.currentTimeMillis()))));
|
||||||
rsrcs.add(resource);
|
rsrcs.add(resource);
|
||||||
ret.setResourceSpecs(rsrcs);
|
ret.setResourceSpecs(rsrcs);
|
||||||
System.out.println(resource);
|
System.out.println(resource);
|
||||||
|
@ -212,7 +212,7 @@ public void testContainerSetup() throws Exception {
|
|||||||
ContainerLaunchContext containerLaunchContext =
|
ContainerLaunchContext containerLaunchContext =
|
||||||
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(file.getAbsolutePath())));
|
.makeQualified(new Path(file.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
rsrc_alpha.setResource(resource_alpha);
|
rsrc_alpha.setResource(resource_alpha);
|
||||||
@ -242,8 +242,8 @@ public void testContainerSetup() throws Exception {
|
|||||||
|
|
||||||
// Now ascertain that the resources are localised correctly.
|
// Now ascertain that the resources are localised correctly.
|
||||||
ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
|
ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
|
||||||
String appIDStr = ConverterUtils.toString(appId);
|
String appIDStr = appId.toString();
|
||||||
String containerIDStr = ConverterUtils.toString(cId);
|
String containerIDStr = cId.toString();
|
||||||
File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
|
File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
|
||||||
File userDir = new File(userCacheDir, user);
|
File userDir = new File(userCacheDir, user);
|
||||||
File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
|
File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
|
||||||
@ -301,7 +301,7 @@ public void testContainerLaunchAndStop() throws IOException,
|
|||||||
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
||||||
|
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
@ -408,7 +408,7 @@ private void testContainerLaunchAndExit(int exitCode) throws IOException,
|
|||||||
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
||||||
|
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
@ -501,7 +501,7 @@ public void testLocalFilesCleanup() throws InterruptedException,
|
|||||||
// containerLaunchContext.resources =
|
// containerLaunchContext.resources =
|
||||||
// new HashMap<CharSequence, LocalResource>();
|
// new HashMap<CharSequence, LocalResource>();
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(FileContext.getLocalFSFileContext()
|
URL.fromPath(FileContext.getLocalFSFileContext()
|
||||||
.makeQualified(new Path(file.getAbsolutePath())));
|
.makeQualified(new Path(file.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
|
LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);
|
||||||
rsrc_alpha.setResource(resource_alpha);
|
rsrc_alpha.setResource(resource_alpha);
|
||||||
@ -534,8 +534,8 @@ public void testLocalFilesCleanup() throws InterruptedException,
|
|||||||
ApplicationState.RUNNING);
|
ApplicationState.RUNNING);
|
||||||
|
|
||||||
// Now ascertain that the resources are localised correctly.
|
// Now ascertain that the resources are localised correctly.
|
||||||
String appIDStr = ConverterUtils.toString(appId);
|
String appIDStr = appId.toString();
|
||||||
String containerIDStr = ConverterUtils.toString(cId);
|
String containerIDStr = cId.toString();
|
||||||
File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
|
File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
|
||||||
File userDir = new File(userCacheDir, user);
|
File userDir = new File(userCacheDir, user);
|
||||||
File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
|
File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
|
||||||
@ -989,7 +989,7 @@ public void testIncreaseContainerResourceWithInvalidResource() throws Exception
|
|||||||
ContainerLaunchContext containerLaunchContext =
|
ContainerLaunchContext containerLaunchContext =
|
||||||
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
@ -1073,7 +1073,7 @@ public void testChangeContainerResource() throws Exception {
|
|||||||
ContainerLaunchContext containerLaunchContext =
|
ContainerLaunchContext containerLaunchContext =
|
||||||
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
@ -1227,7 +1227,7 @@ private void testContainerLaunchAndSignal(SignalContainerCommand command)
|
|||||||
ContainerId cId = createContainerId(0);
|
ContainerId cId = createContainerId(0);
|
||||||
|
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
|
@ -415,7 +415,7 @@ public void testContainerResizeRecovery() throws Exception {
|
|||||||
fileWriter.close();
|
fileWriter.close();
|
||||||
FileContext localFS = FileContext.getLocalFSFileContext();
|
FileContext localFS = FileContext.getLocalFSFileContext();
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha = RecordFactoryProvider
|
LocalResource rsrc_alpha = RecordFactoryProvider
|
||||||
.getRecordFactory(null).newRecordInstance(LocalResource.class);
|
.getRecordFactory(null).newRecordInstance(LocalResource.class);
|
||||||
|
@ -600,7 +600,7 @@ public void testContainerEnvVariables() throws Exception {
|
|||||||
|
|
||||||
// upload the script file so that the container can run it
|
// upload the script file so that the container can run it
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
@ -801,7 +801,7 @@ private void internalKillTest(boolean delayed) throws Exception {
|
|||||||
|
|
||||||
// upload the script file so that the container can run it
|
// upload the script file so that the container can run it
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
@ -1140,7 +1140,7 @@ public void testKillProcessGroup() throws Exception {
|
|||||||
|
|
||||||
// upload the script file so that the container can run it
|
// upload the script file so that the container can run it
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
|
@ -404,7 +404,7 @@ static ResourceLocalizationSpec getMockRsrc(Random r,
|
|||||||
|
|
||||||
when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
|
when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
|
||||||
when(resourceLocalizationSpec.getDestinationDirectory()).
|
when(resourceLocalizationSpec.getDestinationDirectory()).
|
||||||
thenReturn(ConverterUtils.getYarnUrlFromPath(p));
|
thenReturn(URL.fromPath(p));
|
||||||
return resourceLocalizationSpec;
|
return resourceLocalizationSpec;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -24,6 +24,7 @@
|
|||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResourceType;
|
import org.apache.hadoop.yarn.api.records.LocalResourceType;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
||||||
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest;
|
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest;
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
@ -39,8 +40,10 @@ public class TestLocalResource {
|
|||||||
static org.apache.hadoop.yarn.api.records.LocalResource getYarnResource(Path p, long size,
|
static org.apache.hadoop.yarn.api.records.LocalResource getYarnResource(Path p, long size,
|
||||||
long timestamp, LocalResourceType type, LocalResourceVisibility state, String pattern)
|
long timestamp, LocalResourceType type, LocalResourceVisibility state, String pattern)
|
||||||
throws URISyntaxException {
|
throws URISyntaxException {
|
||||||
org.apache.hadoop.yarn.api.records.LocalResource ret = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(org.apache.hadoop.yarn.api.records.LocalResource.class);
|
org.apache.hadoop.yarn.api.records.LocalResource ret =
|
||||||
ret.setResource(ConverterUtils.getYarnUrlFromURI(p.toUri()));
|
RecordFactoryProvider.getRecordFactory(null).newRecordInstance(
|
||||||
|
org.apache.hadoop.yarn.api.records.LocalResource.class);
|
||||||
|
ret.setResource(URL.fromURI(p.toUri()));
|
||||||
ret.setSize(size);
|
ret.setSize(size);
|
||||||
ret.setTimestamp(timestamp);
|
ret.setTimestamp(timestamp);
|
||||||
ret.setType(type);
|
ret.setType(type);
|
||||||
|
@ -945,7 +945,7 @@ public boolean matches(Object o) {
|
|||||||
// Sigh. Thread init of private localizer not accessible
|
// Sigh. Thread init of private localizer not accessible
|
||||||
Thread.sleep(1000);
|
Thread.sleep(1000);
|
||||||
dispatcher.await();
|
dispatcher.await();
|
||||||
String appStr = ConverterUtils.toString(appId);
|
String appStr = appId.toString();
|
||||||
String ctnrStr = c.getContainerId().toString();
|
String ctnrStr = c.getContainerId().toString();
|
||||||
ArgumentCaptor<LocalizerStartContext> contextCaptor = ArgumentCaptor
|
ArgumentCaptor<LocalizerStartContext> contextCaptor = ArgumentCaptor
|
||||||
.forClass(LocalizerStartContext.class);
|
.forClass(LocalizerStartContext.class);
|
||||||
@ -2144,12 +2144,16 @@ public void testParallelDownloadAttemptsForPublicResource() throws Exception {
|
|||||||
// removing pending download request.
|
// removing pending download request.
|
||||||
spyService.getPublicLocalizer().pending.clear();
|
spyService.getPublicLocalizer().pending.clear();
|
||||||
|
|
||||||
|
LocalizerContext lc = mock(LocalizerContext.class);
|
||||||
|
when(lc.getContainerId()).thenReturn(ContainerId.newContainerId(
|
||||||
|
ApplicationAttemptId.newInstance(ApplicationId.newInstance(1L, 1), 1),
|
||||||
|
1L));
|
||||||
|
|
||||||
// Now I need to simulate a race condition wherein Event is added to
|
// Now I need to simulate a race condition wherein Event is added to
|
||||||
// dispatcher before resource state changes to either FAILED or LOCALIZED
|
// dispatcher before resource state changes to either FAILED or LOCALIZED
|
||||||
// Hence sending event directly to dispatcher.
|
// Hence sending event directly to dispatcher.
|
||||||
LocalizerResourceRequestEvent localizerEvent =
|
LocalizerResourceRequestEvent localizerEvent =
|
||||||
new LocalizerResourceRequestEvent(lr, null,
|
new LocalizerResourceRequestEvent(lr, null, lc, null);
|
||||||
mock(LocalizerContext.class), null);
|
|
||||||
|
|
||||||
dispatcher1.getEventHandler().handle(localizerEvent);
|
dispatcher1.getEventHandler().handle(localizerEvent);
|
||||||
// Waiting for download to start. This should return false as new download
|
// Waiting for download to start. This should return false as new download
|
||||||
@ -2457,7 +2461,7 @@ public void testFailedDirsResourceRelease() throws Exception {
|
|||||||
BuilderUtils.newApplicationId(314159265358979L, 3);
|
BuilderUtils.newApplicationId(314159265358979L, 3);
|
||||||
when(app.getUser()).thenReturn(user);
|
when(app.getUser()).thenReturn(user);
|
||||||
when(app.getAppId()).thenReturn(appId);
|
when(app.getAppId()).thenReturn(appId);
|
||||||
when(app.toString()).thenReturn(ConverterUtils.toString(appId));
|
when(app.toString()).thenReturn(appId.toString());
|
||||||
|
|
||||||
// init container.
|
// init container.
|
||||||
final Container c = getMockContainer(appId, 42, user);
|
final Container c = getMockContainer(appId, 42, user);
|
||||||
@ -2468,17 +2472,16 @@ public void testFailedDirsResourceRelease() throws Exception {
|
|||||||
Path usersdir = new Path(tmpDirs.get(i), ContainerLocalizer.USERCACHE);
|
Path usersdir = new Path(tmpDirs.get(i), ContainerLocalizer.USERCACHE);
|
||||||
Path userdir = new Path(usersdir, user);
|
Path userdir = new Path(usersdir, user);
|
||||||
Path allAppsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
|
Path allAppsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
|
||||||
Path appDir = new Path(allAppsdir, ConverterUtils.toString(appId));
|
Path appDir = new Path(allAppsdir, appId.toString());
|
||||||
Path containerDir =
|
Path containerDir =
|
||||||
new Path(appDir, ConverterUtils.toString(c.getContainerId()));
|
new Path(appDir, c.getContainerId().toString());
|
||||||
containerLocalDirs.add(containerDir);
|
containerLocalDirs.add(containerDir);
|
||||||
appLocalDirs.add(appDir);
|
appLocalDirs.add(appDir);
|
||||||
|
|
||||||
Path sysDir =
|
Path sysDir =
|
||||||
new Path(tmpDirs.get(i), ResourceLocalizationService.NM_PRIVATE_DIR);
|
new Path(tmpDirs.get(i), ResourceLocalizationService.NM_PRIVATE_DIR);
|
||||||
Path appSysDir = new Path(sysDir, ConverterUtils.toString(appId));
|
Path appSysDir = new Path(sysDir, appId.toString());
|
||||||
Path containerSysDir =
|
Path containerSysDir = new Path(appSysDir, c.getContainerId().toString());
|
||||||
new Path(appSysDir, ConverterUtils.toString(c.getContainerId()));
|
|
||||||
|
|
||||||
nmLocalContainerDirs.add(containerSysDir);
|
nmLocalContainerDirs.add(containerSysDir);
|
||||||
nmLocalAppDirs.add(appSysDir);
|
nmLocalAppDirs.add(appSysDir);
|
||||||
|
@ -191,7 +191,7 @@ private void verifyLocalFileDeletion(
|
|||||||
|
|
||||||
// AppLogDir should be created
|
// AppLogDir should be created
|
||||||
File app1LogDir =
|
File app1LogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(application1));
|
new File(localLogDir, application1.toString());
|
||||||
app1LogDir.mkdir();
|
app1LogDir.mkdir();
|
||||||
logAggregationService
|
logAggregationService
|
||||||
.handle(new LogHandlerAppStartedEvent(
|
.handle(new LogHandlerAppStartedEvent(
|
||||||
@ -218,7 +218,7 @@ private void verifyLocalFileDeletion(
|
|||||||
verify(delSrvc).delete(eq(user), eq((Path) null),
|
verify(delSrvc).delete(eq(user), eq((Path) null),
|
||||||
eq(new Path(app1LogDir.getAbsolutePath())));
|
eq(new Path(app1LogDir.getAbsolutePath())));
|
||||||
|
|
||||||
String containerIdStr = ConverterUtils.toString(container11);
|
String containerIdStr = container11.toString();
|
||||||
File containerLogDir = new File(app1LogDir, containerIdStr);
|
File containerLogDir = new File(app1LogDir, containerIdStr);
|
||||||
int count = 0;
|
int count = 0;
|
||||||
int maxAttempts = 50;
|
int maxAttempts = 50;
|
||||||
@ -312,7 +312,7 @@ public void testNoLogsUploadedOnAppFinish() throws Exception {
|
|||||||
logAggregationService.start();
|
logAggregationService.start();
|
||||||
|
|
||||||
ApplicationId app = BuilderUtils.newApplicationId(1234, 1);
|
ApplicationId app = BuilderUtils.newApplicationId(1234, 1);
|
||||||
File appLogDir = new File(localLogDir, ConverterUtils.toString(app));
|
File appLogDir = new File(localLogDir, app.toString());
|
||||||
appLogDir.mkdir();
|
appLogDir.mkdir();
|
||||||
LogAggregationContext context =
|
LogAggregationContext context =
|
||||||
LogAggregationContext.newInstance("HOST*", "sys*");
|
LogAggregationContext.newInstance("HOST*", "sys*");
|
||||||
@ -349,7 +349,7 @@ public void testNoContainerOnNode() throws Exception {
|
|||||||
|
|
||||||
// AppLogDir should be created
|
// AppLogDir should be created
|
||||||
File app1LogDir =
|
File app1LogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(application1));
|
new File(localLogDir, application1.toString());
|
||||||
app1LogDir.mkdir();
|
app1LogDir.mkdir();
|
||||||
logAggregationService
|
logAggregationService
|
||||||
.handle(new LogHandlerAppStartedEvent(
|
.handle(new LogHandlerAppStartedEvent(
|
||||||
@ -399,7 +399,7 @@ public void testMultipleAppsLogAggregation() throws Exception {
|
|||||||
|
|
||||||
// AppLogDir should be created
|
// AppLogDir should be created
|
||||||
File app1LogDir =
|
File app1LogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(application1));
|
new File(localLogDir, application1.toString());
|
||||||
app1LogDir.mkdir();
|
app1LogDir.mkdir();
|
||||||
logAggregationService
|
logAggregationService
|
||||||
.handle(new LogHandlerAppStartedEvent(
|
.handle(new LogHandlerAppStartedEvent(
|
||||||
@ -420,7 +420,7 @@ public void testMultipleAppsLogAggregation() throws Exception {
|
|||||||
BuilderUtils.newApplicationAttemptId(application2, 1);
|
BuilderUtils.newApplicationAttemptId(application2, 1);
|
||||||
|
|
||||||
File app2LogDir =
|
File app2LogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(application2));
|
new File(localLogDir, application2.toString());
|
||||||
app2LogDir.mkdir();
|
app2LogDir.mkdir();
|
||||||
LogAggregationContext contextWithAMOnly =
|
LogAggregationContext contextWithAMOnly =
|
||||||
Records.newRecord(LogAggregationContext.class);
|
Records.newRecord(LogAggregationContext.class);
|
||||||
@ -449,7 +449,7 @@ public void testMultipleAppsLogAggregation() throws Exception {
|
|||||||
BuilderUtils.newApplicationAttemptId(application3, 1);
|
BuilderUtils.newApplicationAttemptId(application3, 1);
|
||||||
|
|
||||||
File app3LogDir =
|
File app3LogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(application3));
|
new File(localLogDir, application3.toString());
|
||||||
app3LogDir.mkdir();
|
app3LogDir.mkdir();
|
||||||
LogAggregationContext contextWithAMAndFailed =
|
LogAggregationContext contextWithAMAndFailed =
|
||||||
Records.newRecord(LogAggregationContext.class);
|
Records.newRecord(LogAggregationContext.class);
|
||||||
@ -580,7 +580,7 @@ public void testVerifyAndCreateRemoteDirsFailure()
|
|||||||
BuilderUtils.newApplicationId(System.currentTimeMillis(),
|
BuilderUtils.newApplicationId(System.currentTimeMillis(),
|
||||||
(int) (Math.random() * 1000));
|
(int) (Math.random() * 1000));
|
||||||
File appLogDir =
|
File appLogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(appId2));
|
new File(localLogDir, appId2.toString());
|
||||||
appLogDir.mkdir();
|
appLogDir.mkdir();
|
||||||
logAggregationService.handle(new LogHandlerAppStartedEvent(appId2,
|
logAggregationService.handle(new LogHandlerAppStartedEvent(appId2,
|
||||||
this.user, null, this.acls, contextWithAMAndFailed));
|
this.user, null, this.acls, contextWithAMAndFailed));
|
||||||
@ -755,7 +755,7 @@ public void testLogAggregationCreateDirsFailsWithoutKillingNM()
|
|||||||
(int) (Math.random() * 1000));
|
(int) (Math.random() * 1000));
|
||||||
|
|
||||||
File appLogDir =
|
File appLogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(appId));
|
new File(localLogDir, appId.toString());
|
||||||
appLogDir.mkdir();
|
appLogDir.mkdir();
|
||||||
|
|
||||||
Exception e = new RuntimeException("KABOOM!");
|
Exception e = new RuntimeException("KABOOM!");
|
||||||
@ -802,7 +802,7 @@ public void testLogAggregationCreateDirsFailsWithoutKillingNM()
|
|||||||
private void writeContainerLogs(File appLogDir, ContainerId containerId,
|
private void writeContainerLogs(File appLogDir, ContainerId containerId,
|
||||||
String[] fileName) throws IOException {
|
String[] fileName) throws IOException {
|
||||||
// ContainerLogDir should be created
|
// ContainerLogDir should be created
|
||||||
String containerStr = ConverterUtils.toString(containerId);
|
String containerStr = containerId.toString();
|
||||||
File containerLogDir = new File(appLogDir, containerStr);
|
File containerLogDir = new File(appLogDir, containerStr);
|
||||||
boolean created = containerLogDir.mkdirs();
|
boolean created = containerLogDir.mkdirs();
|
||||||
LOG.info("Created Dir:" + containerLogDir.getAbsolutePath() + " status :"
|
LOG.info("Created Dir:" + containerLogDir.getAbsolutePath() + " status :"
|
||||||
@ -940,7 +940,7 @@ private LogFileStatusInLastCycle verifyContainerLogs(
|
|||||||
Assert.assertTrue("number of containers with logs should be at most " +
|
Assert.assertTrue("number of containers with logs should be at most " +
|
||||||
minNumOfContainers,logMap.size() <= maxNumOfContainers);
|
minNumOfContainers,logMap.size() <= maxNumOfContainers);
|
||||||
for (ContainerId cId : expectedContainerIds) {
|
for (ContainerId cId : expectedContainerIds) {
|
||||||
String containerStr = ConverterUtils.toString(cId);
|
String containerStr = cId.toString();
|
||||||
Map<String, String> thisContainerMap = logMap.remove(containerStr);
|
Map<String, String> thisContainerMap = logMap.remove(containerStr);
|
||||||
Assert.assertEquals(numOfLogsPerContainer, thisContainerMap.size());
|
Assert.assertEquals(numOfLogsPerContainer, thisContainerMap.size());
|
||||||
for (String fileType : logFiles) {
|
for (String fileType : logFiles) {
|
||||||
@ -995,7 +995,7 @@ public void testLogAggregationForRealContainerLaunch() throws IOException,
|
|||||||
ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0);
|
ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0);
|
||||||
|
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
@ -1295,7 +1295,7 @@ public void testLogAggregationServiceWithPatterns() throws Exception {
|
|||||||
// has only logs from stdout and syslog
|
// has only logs from stdout and syslog
|
||||||
// AppLogDir should be created
|
// AppLogDir should be created
|
||||||
File appLogDir1 =
|
File appLogDir1 =
|
||||||
new File(localLogDir, ConverterUtils.toString(application1));
|
new File(localLogDir, application1.toString());
|
||||||
appLogDir1.mkdir();
|
appLogDir1.mkdir();
|
||||||
logAggregationService.handle(new LogHandlerAppStartedEvent(application1,
|
logAggregationService.handle(new LogHandlerAppStartedEvent(application1,
|
||||||
this.user, null, this.acls,
|
this.user, null, this.acls,
|
||||||
@ -1320,7 +1320,7 @@ public void testLogAggregationServiceWithPatterns() throws Exception {
|
|||||||
BuilderUtils.newApplicationAttemptId(application2, 1);
|
BuilderUtils.newApplicationAttemptId(application2, 1);
|
||||||
|
|
||||||
File app2LogDir =
|
File app2LogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(application2));
|
new File(localLogDir, application2.toString());
|
||||||
app2LogDir.mkdir();
|
app2LogDir.mkdir();
|
||||||
LogAggregationContextWithExcludePatterns.setLogAggregationPolicyClassName(
|
LogAggregationContextWithExcludePatterns.setLogAggregationPolicyClassName(
|
||||||
AMOnlyLogAggregationPolicy.class.getName());
|
AMOnlyLogAggregationPolicy.class.getName());
|
||||||
@ -1345,7 +1345,7 @@ public void testLogAggregationServiceWithPatterns() throws Exception {
|
|||||||
ApplicationAttemptId appAttemptId3 =
|
ApplicationAttemptId appAttemptId3 =
|
||||||
BuilderUtils.newApplicationAttemptId(application3, 1);
|
BuilderUtils.newApplicationAttemptId(application3, 1);
|
||||||
File app3LogDir =
|
File app3LogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(application3));
|
new File(localLogDir, application3.toString());
|
||||||
app3LogDir.mkdir();
|
app3LogDir.mkdir();
|
||||||
context1.setLogAggregationPolicyClassName(
|
context1.setLogAggregationPolicyClassName(
|
||||||
AMOnlyLogAggregationPolicy.class.getName());
|
AMOnlyLogAggregationPolicy.class.getName());
|
||||||
@ -1370,7 +1370,7 @@ public void testLogAggregationServiceWithPatterns() throws Exception {
|
|||||||
ApplicationAttemptId appAttemptId4 =
|
ApplicationAttemptId appAttemptId4 =
|
||||||
BuilderUtils.newApplicationAttemptId(application4, 1);
|
BuilderUtils.newApplicationAttemptId(application4, 1);
|
||||||
File app4LogDir =
|
File app4LogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(application4));
|
new File(localLogDir, application4.toString());
|
||||||
app4LogDir.mkdir();
|
app4LogDir.mkdir();
|
||||||
context2.setLogAggregationPolicyClassName(
|
context2.setLogAggregationPolicyClassName(
|
||||||
AMOnlyLogAggregationPolicy.class.getName());
|
AMOnlyLogAggregationPolicy.class.getName());
|
||||||
@ -1872,7 +1872,7 @@ private ContainerId finishContainer(ApplicationId application1,
|
|||||||
containerType);
|
containerType);
|
||||||
// Simulate log-file creation
|
// Simulate log-file creation
|
||||||
File appLogDir1 =
|
File appLogDir1 =
|
||||||
new File(localLogDir, ConverterUtils.toString(application1));
|
new File(localLogDir, application1.toString());
|
||||||
appLogDir1.mkdir();
|
appLogDir1.mkdir();
|
||||||
writeContainerLogs(appLogDir1, containerId, logFiles);
|
writeContainerLogs(appLogDir1, containerId, logFiles);
|
||||||
|
|
||||||
@ -1983,7 +1983,7 @@ private void testLogAggregationService(boolean retentionSizeLimitation)
|
|||||||
|
|
||||||
// AppLogDir should be created
|
// AppLogDir should be created
|
||||||
File appLogDir =
|
File appLogDir =
|
||||||
new File(localLogDir, ConverterUtils.toString(application));
|
new File(localLogDir, application.toString());
|
||||||
appLogDir.mkdir();
|
appLogDir.mkdir();
|
||||||
logAggregationService.handle(new LogHandlerAppStartedEvent(application,
|
logAggregationService.handle(new LogHandlerAppStartedEvent(application,
|
||||||
this.user, null, this.acls, logAggregationContextWithInterval));
|
this.user, null, this.acls, logAggregationContextWithInterval));
|
||||||
|
@ -210,7 +210,7 @@ public void testContainerKillOnMemoryOverflow() throws IOException,
|
|||||||
ContainerId cId = ContainerId.newContainerId(appAttemptId, 0);
|
ContainerId cId = ContainerId.newContainerId(appAttemptId, 0);
|
||||||
|
|
||||||
URL resource_alpha =
|
URL resource_alpha =
|
||||||
ConverterUtils.getYarnUrlFromPath(localFS
|
URL.fromPath(localFS
|
||||||
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
.makeQualified(new Path(scriptFile.getAbsolutePath())));
|
||||||
LocalResource rsrc_alpha =
|
LocalResource rsrc_alpha =
|
||||||
recordFactory.newRecordInstance(LocalResource.class);
|
recordFactory.newRecordInstance(LocalResource.class);
|
||||||
|
@ -350,7 +350,7 @@ public void testStartResourceLocalization() throws IOException {
|
|||||||
Path appRsrcPath = new Path("hdfs://some/app/resource");
|
Path appRsrcPath = new Path("hdfs://some/app/resource");
|
||||||
LocalResourcePBImpl rsrcPb = (LocalResourcePBImpl)
|
LocalResourcePBImpl rsrcPb = (LocalResourcePBImpl)
|
||||||
LocalResource.newInstance(
|
LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(appRsrcPath),
|
URL.fromPath(appRsrcPath),
|
||||||
LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION,
|
LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION,
|
||||||
123L, 456L);
|
123L, 456L);
|
||||||
LocalResourceProto appRsrcProto = rsrcPb.getProto();
|
LocalResourceProto appRsrcProto = rsrcPb.getProto();
|
||||||
@ -383,7 +383,7 @@ public void testStartResourceLocalization() throws IOException {
|
|||||||
// start some public and private resources
|
// start some public and private resources
|
||||||
Path pubRsrcPath1 = new Path("hdfs://some/public/resource1");
|
Path pubRsrcPath1 = new Path("hdfs://some/public/resource1");
|
||||||
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(pubRsrcPath1),
|
URL.fromPath(pubRsrcPath1),
|
||||||
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
||||||
789L, 135L);
|
789L, 135L);
|
||||||
LocalResourceProto pubRsrcProto1 = rsrcPb.getProto();
|
LocalResourceProto pubRsrcProto1 = rsrcPb.getProto();
|
||||||
@ -392,7 +392,7 @@ public void testStartResourceLocalization() throws IOException {
|
|||||||
pubRsrcLocalPath1);
|
pubRsrcLocalPath1);
|
||||||
Path pubRsrcPath2 = new Path("hdfs://some/public/resource2");
|
Path pubRsrcPath2 = new Path("hdfs://some/public/resource2");
|
||||||
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(pubRsrcPath2),
|
URL.fromPath(pubRsrcPath2),
|
||||||
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
||||||
789L, 135L);
|
789L, 135L);
|
||||||
LocalResourceProto pubRsrcProto2 = rsrcPb.getProto();
|
LocalResourceProto pubRsrcProto2 = rsrcPb.getProto();
|
||||||
@ -401,7 +401,7 @@ public void testStartResourceLocalization() throws IOException {
|
|||||||
pubRsrcLocalPath2);
|
pubRsrcLocalPath2);
|
||||||
Path privRsrcPath = new Path("hdfs://some/private/resource");
|
Path privRsrcPath = new Path("hdfs://some/private/resource");
|
||||||
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(privRsrcPath),
|
URL.fromPath(privRsrcPath),
|
||||||
LocalResourceType.PATTERN, LocalResourceVisibility.PRIVATE,
|
LocalResourceType.PATTERN, LocalResourceVisibility.PRIVATE,
|
||||||
789L, 680L, "*pattern*");
|
789L, 680L, "*pattern*");
|
||||||
LocalResourceProto privRsrcProto = rsrcPb.getProto();
|
LocalResourceProto privRsrcProto = rsrcPb.getProto();
|
||||||
@ -446,7 +446,7 @@ public void testFinishResourceLocalization() throws IOException {
|
|||||||
Path appRsrcPath = new Path("hdfs://some/app/resource");
|
Path appRsrcPath = new Path("hdfs://some/app/resource");
|
||||||
LocalResourcePBImpl rsrcPb = (LocalResourcePBImpl)
|
LocalResourcePBImpl rsrcPb = (LocalResourcePBImpl)
|
||||||
LocalResource.newInstance(
|
LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(appRsrcPath),
|
URL.fromPath(appRsrcPath),
|
||||||
LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION,
|
LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION,
|
||||||
123L, 456L);
|
123L, 456L);
|
||||||
LocalResourceProto appRsrcProto = rsrcPb.getProto();
|
LocalResourceProto appRsrcProto = rsrcPb.getProto();
|
||||||
@ -486,7 +486,7 @@ public void testFinishResourceLocalization() throws IOException {
|
|||||||
// start some public and private resources
|
// start some public and private resources
|
||||||
Path pubRsrcPath1 = new Path("hdfs://some/public/resource1");
|
Path pubRsrcPath1 = new Path("hdfs://some/public/resource1");
|
||||||
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(pubRsrcPath1),
|
URL.fromPath(pubRsrcPath1),
|
||||||
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
||||||
789L, 135L);
|
789L, 135L);
|
||||||
LocalResourceProto pubRsrcProto1 = rsrcPb.getProto();
|
LocalResourceProto pubRsrcProto1 = rsrcPb.getProto();
|
||||||
@ -495,7 +495,7 @@ public void testFinishResourceLocalization() throws IOException {
|
|||||||
pubRsrcLocalPath1);
|
pubRsrcLocalPath1);
|
||||||
Path pubRsrcPath2 = new Path("hdfs://some/public/resource2");
|
Path pubRsrcPath2 = new Path("hdfs://some/public/resource2");
|
||||||
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(pubRsrcPath2),
|
URL.fromPath(pubRsrcPath2),
|
||||||
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
||||||
789L, 135L);
|
789L, 135L);
|
||||||
LocalResourceProto pubRsrcProto2 = rsrcPb.getProto();
|
LocalResourceProto pubRsrcProto2 = rsrcPb.getProto();
|
||||||
@ -504,7 +504,7 @@ public void testFinishResourceLocalization() throws IOException {
|
|||||||
pubRsrcLocalPath2);
|
pubRsrcLocalPath2);
|
||||||
Path privRsrcPath = new Path("hdfs://some/private/resource");
|
Path privRsrcPath = new Path("hdfs://some/private/resource");
|
||||||
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(privRsrcPath),
|
URL.fromPath(privRsrcPath),
|
||||||
LocalResourceType.PATTERN, LocalResourceVisibility.PRIVATE,
|
LocalResourceType.PATTERN, LocalResourceVisibility.PRIVATE,
|
||||||
789L, 680L, "*pattern*");
|
789L, 680L, "*pattern*");
|
||||||
LocalResourceProto privRsrcProto = rsrcPb.getProto();
|
LocalResourceProto privRsrcProto = rsrcPb.getProto();
|
||||||
@ -565,7 +565,7 @@ public void testRemoveLocalizedResource() throws IOException {
|
|||||||
Path appRsrcPath = new Path("hdfs://some/app/resource");
|
Path appRsrcPath = new Path("hdfs://some/app/resource");
|
||||||
LocalResourcePBImpl rsrcPb = (LocalResourcePBImpl)
|
LocalResourcePBImpl rsrcPb = (LocalResourcePBImpl)
|
||||||
LocalResource.newInstance(
|
LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(appRsrcPath),
|
URL.fromPath(appRsrcPath),
|
||||||
LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION,
|
LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION,
|
||||||
123L, 456L);
|
123L, 456L);
|
||||||
LocalResourceProto appRsrcProto = rsrcPb.getProto();
|
LocalResourceProto appRsrcProto = rsrcPb.getProto();
|
||||||
@ -595,7 +595,7 @@ public void testRemoveLocalizedResource() throws IOException {
|
|||||||
// add public and private resources and remove some
|
// add public and private resources and remove some
|
||||||
Path pubRsrcPath1 = new Path("hdfs://some/public/resource1");
|
Path pubRsrcPath1 = new Path("hdfs://some/public/resource1");
|
||||||
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(pubRsrcPath1),
|
URL.fromPath(pubRsrcPath1),
|
||||||
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
||||||
789L, 135L);
|
789L, 135L);
|
||||||
LocalResourceProto pubRsrcProto1 = rsrcPb.getProto();
|
LocalResourceProto pubRsrcProto1 = rsrcPb.getProto();
|
||||||
@ -611,7 +611,7 @@ public void testRemoveLocalizedResource() throws IOException {
|
|||||||
stateStore.finishResourceLocalization(null, null, pubLocalizedProto1);
|
stateStore.finishResourceLocalization(null, null, pubLocalizedProto1);
|
||||||
Path pubRsrcPath2 = new Path("hdfs://some/public/resource2");
|
Path pubRsrcPath2 = new Path("hdfs://some/public/resource2");
|
||||||
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(pubRsrcPath2),
|
URL.fromPath(pubRsrcPath2),
|
||||||
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
LocalResourceType.FILE, LocalResourceVisibility.PUBLIC,
|
||||||
789L, 135L);
|
789L, 135L);
|
||||||
LocalResourceProto pubRsrcProto2 = rsrcPb.getProto();
|
LocalResourceProto pubRsrcProto2 = rsrcPb.getProto();
|
||||||
@ -628,7 +628,7 @@ public void testRemoveLocalizedResource() throws IOException {
|
|||||||
stateStore.removeLocalizedResource(null, null, pubRsrcLocalPath2);
|
stateStore.removeLocalizedResource(null, null, pubRsrcLocalPath2);
|
||||||
Path privRsrcPath = new Path("hdfs://some/private/resource");
|
Path privRsrcPath = new Path("hdfs://some/private/resource");
|
||||||
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
rsrcPb = (LocalResourcePBImpl) LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromPath(privRsrcPath),
|
URL.fromPath(privRsrcPath),
|
||||||
LocalResourceType.PATTERN, LocalResourceVisibility.PRIVATE,
|
LocalResourceType.PATTERN, LocalResourceVisibility.PRIVATE,
|
||||||
789L, 680L, "*pattern*");
|
789L, 680L, "*pattern*");
|
||||||
LocalResourceProto privRsrcProto = rsrcPb.getProto();
|
LocalResourceProto privRsrcProto = rsrcPb.getProto();
|
||||||
|
@ -249,7 +249,7 @@ private void writeContainerLogs(Context nmContext,
|
|||||||
containerLogDir.mkdirs();
|
containerLogDir.mkdirs();
|
||||||
for (String fileType : new String[] { "stdout", "stderr", "syslog" }) {
|
for (String fileType : new String[] { "stdout", "stderr", "syslog" }) {
|
||||||
Writer writer = new FileWriter(new File(containerLogDir, fileType));
|
Writer writer = new FileWriter(new File(containerLogDir, fileType));
|
||||||
writer.write(ConverterUtils.toString(containerId) + "\n Hello "
|
writer.write(containerId.toString() + "\n Hello "
|
||||||
+ fileType + "!");
|
+ fileType + "!");
|
||||||
writer.close();
|
writer.close();
|
||||||
}
|
}
|
||||||
|
@ -40,6 +40,7 @@
|
|||||||
import org.apache.hadoop.fs.FileUtil;
|
import org.apache.hadoop.fs.FileUtil;
|
||||||
import org.apache.hadoop.util.NodeHealthScriptRunner;
|
import org.apache.hadoop.util.NodeHealthScriptRunner;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.event.AsyncDispatcher;
|
import org.apache.hadoop.yarn.event.AsyncDispatcher;
|
||||||
@ -280,7 +281,7 @@ public void testNodeHelper(String path, String media) throws JSONException,
|
|||||||
verifyNodeContainerInfo(
|
verifyNodeContainerInfo(
|
||||||
conInfo.getJSONObject(i),
|
conInfo.getJSONObject(i),
|
||||||
nmContext.getContainers().get(
|
nmContext.getContainers().get(
|
||||||
ConverterUtils.toContainerId(conInfo.getJSONObject(i).getString(
|
ContainerId.fromString(conInfo.getJSONObject(i).getString(
|
||||||
"id"))));
|
"id"))));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -316,7 +317,7 @@ public void testNodeSingleContainersHelper(String media)
|
|||||||
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
|
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
|
||||||
JSONObject json = response.getEntity(JSONObject.class);
|
JSONObject json = response.getEntity(JSONObject.class);
|
||||||
verifyNodeContainerInfo(json.getJSONObject("container"), nmContext
|
verifyNodeContainerInfo(json.getJSONObject("container"), nmContext
|
||||||
.getContainers().get(ConverterUtils.toContainerId(id)));
|
.getContainers().get(ContainerId.fromString(id)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -449,7 +450,7 @@ public void testNodeSingleContainerXML() throws JSONException, Exception {
|
|||||||
NodeList nodes = dom.getElementsByTagName("container");
|
NodeList nodes = dom.getElementsByTagName("container");
|
||||||
assertEquals("incorrect number of elements", 1, nodes.getLength());
|
assertEquals("incorrect number of elements", 1, nodes.getLength());
|
||||||
verifyContainersInfoXML(nodes,
|
verifyContainersInfoXML(nodes,
|
||||||
nmContext.getContainers().get(ConverterUtils.toContainerId(id)));
|
nmContext.getContainers().get(ContainerId.fromString(id)));
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1310,7 +1310,7 @@ private static void removeApplication(Configuration conf, String applicationId)
|
|||||||
rmStore.init(conf);
|
rmStore.init(conf);
|
||||||
rmStore.start();
|
rmStore.start();
|
||||||
try {
|
try {
|
||||||
ApplicationId removeAppId = ConverterUtils.toApplicationId(applicationId);
|
ApplicationId removeAppId = ApplicationId.fromString(applicationId);
|
||||||
LOG.info("Deleting application " + removeAppId + " from state store");
|
LOG.info("Deleting application " + removeAppId + " from state store");
|
||||||
rmStore.removeApplication(removeAppId);
|
rmStore.removeApplication(removeAppId);
|
||||||
LOG.info("Application is deleted from state store");
|
LOG.info("Application is deleted from state store");
|
||||||
|
@ -499,7 +499,7 @@ private int loadRMApp(RMState rmState, LeveldbIterator iter, String appIdStr,
|
|||||||
|
|
||||||
private ApplicationStateData createApplicationState(String appIdStr,
|
private ApplicationStateData createApplicationState(String appIdStr,
|
||||||
byte[] data) throws IOException {
|
byte[] data) throws IOException {
|
||||||
ApplicationId appId = ConverterUtils.toApplicationId(appIdStr);
|
ApplicationId appId = ApplicationId.fromString(appIdStr);
|
||||||
ApplicationStateDataPBImpl appState =
|
ApplicationStateDataPBImpl appState =
|
||||||
new ApplicationStateDataPBImpl(
|
new ApplicationStateDataPBImpl(
|
||||||
ApplicationStateDataProto.parseFrom(data));
|
ApplicationStateDataProto.parseFrom(data));
|
||||||
@ -529,8 +529,7 @@ ApplicationStateData loadRMAppState(ApplicationId appId) throws IOException {
|
|||||||
|
|
||||||
private ApplicationAttemptStateData createAttemptState(String itemName,
|
private ApplicationAttemptStateData createAttemptState(String itemName,
|
||||||
byte[] data) throws IOException {
|
byte[] data) throws IOException {
|
||||||
ApplicationAttemptId attemptId =
|
ApplicationAttemptId attemptId = ApplicationAttemptId.fromString(itemName);
|
||||||
ConverterUtils.toApplicationAttemptId(itemName);
|
|
||||||
ApplicationAttemptStateDataPBImpl attemptState =
|
ApplicationAttemptStateDataPBImpl attemptState =
|
||||||
new ApplicationAttemptStateDataPBImpl(
|
new ApplicationAttemptStateDataPBImpl(
|
||||||
ApplicationAttemptStateDataProto.parseFrom(data));
|
ApplicationAttemptStateDataProto.parseFrom(data));
|
||||||
|
@ -543,7 +543,7 @@ private synchronized void loadRMAppState(RMState rmState) throws Exception {
|
|||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
LOG.debug("Loading application from znode: " + childNodeName);
|
LOG.debug("Loading application from znode: " + childNodeName);
|
||||||
}
|
}
|
||||||
ApplicationId appId = ConverterUtils.toApplicationId(childNodeName);
|
ApplicationId appId = ApplicationId.fromString(childNodeName);
|
||||||
ApplicationStateDataPBImpl appState =
|
ApplicationStateDataPBImpl appState =
|
||||||
new ApplicationStateDataPBImpl(
|
new ApplicationStateDataPBImpl(
|
||||||
ApplicationStateDataProto.parseFrom(childData));
|
ApplicationStateDataProto.parseFrom(childData));
|
||||||
|
@ -133,7 +133,7 @@ public Map<NodeId, ResourceOption> getNodeResourceMap() {
|
|||||||
= new HashMap<NodeId, ResourceOption> ();
|
= new HashMap<NodeId, ResourceOption> ();
|
||||||
|
|
||||||
for (String node : nodes) {
|
for (String node : nodes) {
|
||||||
NodeId nid = ConverterUtils.toNodeId(node);
|
NodeId nid = NodeId.fromString(node);
|
||||||
int vcores = getVcoresPerNode(node);
|
int vcores = getVcoresPerNode(node);
|
||||||
int memory = getMemoryPerNode(node);
|
int memory = getMemoryPerNode(node);
|
||||||
int overCommitTimeout = getOverCommitTimeoutPerNode(node);
|
int overCommitTimeout = getOverCommitTimeoutPerNode(node);
|
||||||
|
@ -345,7 +345,7 @@ public String getLogURL() {
|
|||||||
logURL.append(WebAppUtils.getHttpSchemePrefix(rmContext
|
logURL.append(WebAppUtils.getHttpSchemePrefix(rmContext
|
||||||
.getYarnConfiguration()));
|
.getYarnConfiguration()));
|
||||||
logURL.append(WebAppUtils.getRunningLogURL(
|
logURL.append(WebAppUtils.getRunningLogURL(
|
||||||
container.getNodeHttpAddress(), ConverterUtils.toString(containerId),
|
container.getNodeHttpAddress(), containerId.toString(),
|
||||||
user));
|
user));
|
||||||
return logURL.toString();
|
return logURL.toString();
|
||||||
} finally {
|
} finally {
|
||||||
|
@ -82,8 +82,8 @@ protected void renderData(Block html) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
AppInfo app = new AppInfo(appReport);
|
AppInfo app = new AppInfo(appReport);
|
||||||
ApplicationAttemptId appAttemptId =
|
ApplicationAttemptId appAttemptId = ApplicationAttemptId.fromString(
|
||||||
ConverterUtils.toApplicationAttemptId(app.getCurrentAppAttemptId());
|
app.getCurrentAppAttemptId());
|
||||||
String queuePercent = "N/A";
|
String queuePercent = "N/A";
|
||||||
String clusterPercent = "N/A";
|
String clusterPercent = "N/A";
|
||||||
if(appReport.getApplicationResourceUsageReport() != null) {
|
if(appReport.getApplicationResourceUsageReport() != null) {
|
||||||
|
@ -220,7 +220,7 @@ private String ahsRedirectPath(String uri, RMWebApp rmWebApp) {
|
|||||||
break;
|
break;
|
||||||
case "appattempt":
|
case "appattempt":
|
||||||
try{
|
try{
|
||||||
appAttemptId = ConverterUtils.toApplicationAttemptId(parts[3]);
|
appAttemptId = ApplicationAttemptId.fromString(parts[3]);
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
LOG.debug("Error parsing {} as an ApplicationAttemptId",
|
LOG.debug("Error parsing {} as an ApplicationAttemptId",
|
||||||
parts[3], e);
|
parts[3], e);
|
||||||
|
@ -109,6 +109,7 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
|
import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
|
||||||
import org.apache.hadoop.yarn.api.records.ReservationRequests;
|
import org.apache.hadoop.yarn.api.records.ReservationRequests;
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
@ -368,7 +369,7 @@ public NodeInfo getNode(@PathParam("nodeId") String nodeId) {
|
|||||||
if (sched == null) {
|
if (sched == null) {
|
||||||
throw new NotFoundException("Null ResourceScheduler instance");
|
throw new NotFoundException("Null ResourceScheduler instance");
|
||||||
}
|
}
|
||||||
NodeId nid = ConverterUtils.toNodeId(nodeId);
|
NodeId nid = NodeId.fromString(nodeId);
|
||||||
RMNode ni = this.rm.getRMContext().getRMNodes().get(nid);
|
RMNode ni = this.rm.getRMContext().getRMNodes().get(nid);
|
||||||
boolean isInactive = false;
|
boolean isInactive = false;
|
||||||
if (ni == null) {
|
if (ni == null) {
|
||||||
@ -1461,9 +1462,7 @@ protected ApplicationSubmissionContext createAppSubmissionContext(
|
|||||||
String error =
|
String error =
|
||||||
"Could not parse application id " + newApp.getApplicationId();
|
"Could not parse application id " + newApp.getApplicationId();
|
||||||
try {
|
try {
|
||||||
appid =
|
appid = ApplicationId.fromString(newApp.getApplicationId());
|
||||||
ConverterUtils.toApplicationId(recordFactory,
|
|
||||||
newApp.getApplicationId());
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new BadRequestException(error);
|
throw new BadRequestException(error);
|
||||||
}
|
}
|
||||||
@ -1536,7 +1535,7 @@ protected ContainerLaunchContext createContainerLaunchContext(
|
|||||||
LocalResourceInfo l = entry.getValue();
|
LocalResourceInfo l = entry.getValue();
|
||||||
LocalResource lr =
|
LocalResource lr =
|
||||||
LocalResource.newInstance(
|
LocalResource.newInstance(
|
||||||
ConverterUtils.getYarnUrlFromURI(l.getUrl()), l.getType(),
|
URL.fromURI(l.getUrl()), l.getType(),
|
||||||
l.getVisibility(), l.getSize(), l.getTimestamp());
|
l.getVisibility(), l.getSize(), l.getTimestamp());
|
||||||
hlr.put(entry.getKey(), lr);
|
hlr.put(entry.getKey(), lr);
|
||||||
}
|
}
|
||||||
|
@ -63,7 +63,7 @@ public AppAttemptInfo(ResourceManager rm, RMAppAttempt attempt, String user,
|
|||||||
this.nodeId = masterContainer.getNodeId().toString();
|
this.nodeId = masterContainer.getNodeId().toString();
|
||||||
this.logsLink = WebAppUtils.getRunningLogURL(schemePrefix
|
this.logsLink = WebAppUtils.getRunningLogURL(schemePrefix
|
||||||
+ masterContainer.getNodeHttpAddress(),
|
+ masterContainer.getNodeHttpAddress(),
|
||||||
ConverterUtils.toString(masterContainer.getId()), user);
|
masterContainer.getId().toString(), user);
|
||||||
if (rm.getResourceScheduler() instanceof AbstractYarnScheduler) {
|
if (rm.getResourceScheduler() instanceof AbstractYarnScheduler) {
|
||||||
AbstractYarnScheduler ayScheduler =
|
AbstractYarnScheduler ayScheduler =
|
||||||
(AbstractYarnScheduler) rm.getResourceScheduler();
|
(AbstractYarnScheduler) rm.getResourceScheduler();
|
||||||
|
@ -165,8 +165,7 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
|
|||||||
this.amContainerLogsExist = true;
|
this.amContainerLogsExist = true;
|
||||||
this.amContainerLogs = WebAppUtils.getRunningLogURL(
|
this.amContainerLogs = WebAppUtils.getRunningLogURL(
|
||||||
schemePrefix + masterContainer.getNodeHttpAddress(),
|
schemePrefix + masterContainer.getNodeHttpAddress(),
|
||||||
ConverterUtils.toString(masterContainer.getId()),
|
masterContainer.getId().toString(), app.getUser());
|
||||||
app.getUser());
|
|
||||||
this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
|
this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -218,7 +218,7 @@ public void testRefreshNodesResourceWithFileSystemBasedConfigurationProvider()
|
|||||||
fail("Should not get any exceptions");
|
fail("Should not get any exceptions");
|
||||||
}
|
}
|
||||||
|
|
||||||
NodeId nid = ConverterUtils.toNodeId("h1:1234");
|
NodeId nid = NodeId.fromString("h1:1234");
|
||||||
RMNode ni = rm.getRMContext().getRMNodes().get(nid);
|
RMNode ni = rm.getRMContext().getRMNodes().get(nid);
|
||||||
Resource resource = ni.getTotalCapability();
|
Resource resource = ni.getTotalCapability();
|
||||||
Assert.assertEquals("<memory:5120, vCores:5>", resource.toString());
|
Assert.assertEquals("<memory:5120, vCores:5>", resource.toString());
|
||||||
@ -256,7 +256,7 @@ public void testResourcePersistentForNMRegistrationWithNewResource()
|
|||||||
fail("Should not get any exceptions");
|
fail("Should not get any exceptions");
|
||||||
}
|
}
|
||||||
|
|
||||||
NodeId nid = ConverterUtils.toNodeId("h1:1234");
|
NodeId nid = NodeId.fromString("h1:1234");
|
||||||
RMNode ni = rm.getRMContext().getRMNodes().get(nid);
|
RMNode ni = rm.getRMContext().getRMNodes().get(nid);
|
||||||
Resource resource = ni.getTotalCapability();
|
Resource resource = ni.getTotalCapability();
|
||||||
Assert.assertEquals("<memory:5120, vCores:5>", resource.toString());
|
Assert.assertEquals("<memory:5120, vCores:5>", resource.toString());
|
||||||
|
@ -181,7 +181,7 @@ protected ContainerId storeAttempt(RMStateStore store,
|
|||||||
RMAppAttemptMetrics mockRmAppAttemptMetrics =
|
RMAppAttemptMetrics mockRmAppAttemptMetrics =
|
||||||
mock(RMAppAttemptMetrics.class);
|
mock(RMAppAttemptMetrics.class);
|
||||||
Container container = new ContainerPBImpl();
|
Container container = new ContainerPBImpl();
|
||||||
container.setId(ConverterUtils.toContainerId(containerIdStr));
|
container.setId(ContainerId.fromString(containerIdStr));
|
||||||
RMAppAttempt mockAttempt = mock(RMAppAttempt.class);
|
RMAppAttempt mockAttempt = mock(RMAppAttempt.class);
|
||||||
when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
|
when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
|
||||||
when(mockAttempt.getMasterContainer()).thenReturn(container);
|
when(mockAttempt.getMasterContainer()).thenReturn(container);
|
||||||
@ -225,8 +225,8 @@ void testRMAppStateStore(RMStateStoreHelper stateStoreHelper,
|
|||||||
ClientToAMTokenSecretManagerInRM clientToAMTokenMgr =
|
ClientToAMTokenSecretManagerInRM clientToAMTokenMgr =
|
||||||
new ClientToAMTokenSecretManagerInRM();
|
new ClientToAMTokenSecretManagerInRM();
|
||||||
|
|
||||||
ApplicationAttemptId attemptId1 = ConverterUtils
|
ApplicationAttemptId attemptId1 = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId("appattempt_1352994193343_0001_000001");
|
"appattempt_1352994193343_0001_000001");
|
||||||
ApplicationId appId1 = attemptId1.getApplicationId();
|
ApplicationId appId1 = attemptId1.getApplicationId();
|
||||||
storeApp(store, appId1, submitTime, startTime);
|
storeApp(store, appId1, submitTime, startTime);
|
||||||
verifier.afterStoreApp(store, appId1);
|
verifier.afterStoreApp(store, appId1);
|
||||||
@ -242,8 +242,8 @@ void testRMAppStateStore(RMStateStoreHelper stateStoreHelper,
|
|||||||
appAttemptToken1, clientTokenKey1, dispatcher);
|
appAttemptToken1, clientTokenKey1, dispatcher);
|
||||||
|
|
||||||
String appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
|
String appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
|
||||||
ApplicationAttemptId attemptId2 =
|
ApplicationAttemptId attemptId2 = ApplicationAttemptId.fromString(
|
||||||
ConverterUtils.toApplicationAttemptId(appAttemptIdStr2);
|
appAttemptIdStr2);
|
||||||
|
|
||||||
// create application token and client token key for attempt2
|
// create application token and client token key for attempt2
|
||||||
Token<AMRMTokenIdentifier> appAttemptToken2 =
|
Token<AMRMTokenIdentifier> appAttemptToken2 =
|
||||||
@ -255,8 +255,8 @@ void testRMAppStateStore(RMStateStoreHelper stateStoreHelper,
|
|||||||
"container_1352994193343_0001_02_000001",
|
"container_1352994193343_0001_02_000001",
|
||||||
appAttemptToken2, clientTokenKey2, dispatcher);
|
appAttemptToken2, clientTokenKey2, dispatcher);
|
||||||
|
|
||||||
ApplicationAttemptId attemptIdRemoved = ConverterUtils
|
ApplicationAttemptId attemptIdRemoved = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId("appattempt_1352994193343_0002_000001");
|
"appattempt_1352994193343_0002_000001");
|
||||||
ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
|
ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
|
||||||
storeApp(store, appIdRemoved, submitTime, startTime);
|
storeApp(store, appIdRemoved, submitTime, startTime);
|
||||||
storeAttempt(store, attemptIdRemoved,
|
storeAttempt(store, attemptIdRemoved,
|
||||||
|
@ -167,7 +167,7 @@ public void testFSRMStateStore() throws Exception {
|
|||||||
(FileSystemRMStateStore) fsTester.getRMStateStore();
|
(FileSystemRMStateStore) fsTester.getRMStateStore();
|
||||||
String appAttemptIdStr3 = "appattempt_1352994193343_0001_000003";
|
String appAttemptIdStr3 = "appattempt_1352994193343_0001_000003";
|
||||||
ApplicationAttemptId attemptId3 =
|
ApplicationAttemptId attemptId3 =
|
||||||
ConverterUtils.toApplicationAttemptId(appAttemptIdStr3);
|
ApplicationAttemptId.fromString(appAttemptIdStr3);
|
||||||
Path appDir =
|
Path appDir =
|
||||||
fsTester.store.getAppDir(attemptId3.getApplicationId().toString());
|
fsTester.store.getAppDir(attemptId3.getApplicationId().toString());
|
||||||
Path tempAppAttemptFile =
|
Path tempAppAttemptFile =
|
||||||
@ -347,7 +347,7 @@ protected void modifyAppState() throws Exception {
|
|||||||
// imitate appAttemptFile1 is still .new, but old one is deleted
|
// imitate appAttemptFile1 is still .new, but old one is deleted
|
||||||
String appAttemptIdStr1 = "appattempt_1352994193343_0001_000001";
|
String appAttemptIdStr1 = "appattempt_1352994193343_0001_000001";
|
||||||
ApplicationAttemptId attemptId1 =
|
ApplicationAttemptId attemptId1 =
|
||||||
ConverterUtils.toApplicationAttemptId(appAttemptIdStr1);
|
ApplicationAttemptId.fromString(appAttemptIdStr1);
|
||||||
Path appDir =
|
Path appDir =
|
||||||
fsTester.store.getAppDir(attemptId1.getApplicationId().toString());
|
fsTester.store.getAppDir(attemptId1.getApplicationId().toString());
|
||||||
Path appAttemptFile1 =
|
Path appAttemptFile1 =
|
||||||
|
@ -34,6 +34,7 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||||
import org.apache.hadoop.yarn.api.records.Container;
|
import org.apache.hadoop.yarn.api.records.Container;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
|
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
|
||||||
@ -386,14 +387,14 @@ public void testFencedState() throws Exception {
|
|||||||
// Add a new attempt
|
// Add a new attempt
|
||||||
ClientToAMTokenSecretManagerInRM clientToAMTokenMgr =
|
ClientToAMTokenSecretManagerInRM clientToAMTokenMgr =
|
||||||
new ClientToAMTokenSecretManagerInRM();
|
new ClientToAMTokenSecretManagerInRM();
|
||||||
ApplicationAttemptId attemptId = ConverterUtils
|
ApplicationAttemptId attemptId = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId("appattempt_1234567894321_0001_000001");
|
"appattempt_1234567894321_0001_000001");
|
||||||
SecretKey clientTokenMasterKey =
|
SecretKey clientTokenMasterKey =
|
||||||
clientToAMTokenMgr.createMasterKey(attemptId);
|
clientToAMTokenMgr.createMasterKey(attemptId);
|
||||||
RMAppAttemptMetrics mockRmAppAttemptMetrics =
|
RMAppAttemptMetrics mockRmAppAttemptMetrics =
|
||||||
mock(RMAppAttemptMetrics.class);
|
mock(RMAppAttemptMetrics.class);
|
||||||
Container container = new ContainerPBImpl();
|
Container container = new ContainerPBImpl();
|
||||||
container.setId(ConverterUtils.toContainerId("container_1234567891234_0001_01_000001"));
|
container.setId(ContainerId.fromString("container_1234567891234_0001_01_000001"));
|
||||||
RMAppAttempt mockAttempt = mock(RMAppAttempt.class);
|
RMAppAttempt mockAttempt = mock(RMAppAttempt.class);
|
||||||
when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
|
when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
|
||||||
when(mockAttempt.getMasterContainer()).thenReturn(container);
|
when(mockAttempt.getMasterContainer()).thenReturn(container);
|
||||||
@ -478,8 +479,8 @@ public void testDuplicateRMAppDeletion() throws Exception {
|
|||||||
TestDispatcher dispatcher = new TestDispatcher();
|
TestDispatcher dispatcher = new TestDispatcher();
|
||||||
store.setRMDispatcher(dispatcher);
|
store.setRMDispatcher(dispatcher);
|
||||||
|
|
||||||
ApplicationAttemptId attemptIdRemoved = ConverterUtils
|
ApplicationAttemptId attemptIdRemoved = ApplicationAttemptId.fromString(
|
||||||
.toApplicationAttemptId("appattempt_1352994193343_0002_000001");
|
"appattempt_1352994193343_0002_000001");
|
||||||
ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
|
ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
|
||||||
storeApp(store, appIdRemoved, submitTime, startTime);
|
storeApp(store, appIdRemoved, submitTime, startTime);
|
||||||
storeAttempt(store, attemptIdRemoved,
|
storeAttempt(store, attemptIdRemoved,
|
||||||
|
@ -18,32 +18,21 @@
|
|||||||
|
|
||||||
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
|
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
|
||||||
|
|
||||||
import static org.junit.Assert.assertEquals;
|
import com.google.inject.Guice;
|
||||||
import static org.junit.Assert.assertTrue;
|
import com.google.inject.Injector;
|
||||||
import static org.junit.Assume.assumeTrue;
|
import com.google.inject.Singleton;
|
||||||
|
import com.google.inject.servlet.GuiceServletContextListener;
|
||||||
import java.io.*;
|
import com.google.inject.servlet.ServletModule;
|
||||||
import java.net.URI;
|
import com.sun.jersey.api.client.Client;
|
||||||
import java.util.ArrayList;
|
import com.sun.jersey.api.client.ClientResponse;
|
||||||
import java.util.Arrays;
|
import com.sun.jersey.api.client.ClientResponse.Status;
|
||||||
import java.util.Collection;
|
import com.sun.jersey.api.client.WebResource;
|
||||||
import java.util.Enumeration;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Properties;
|
|
||||||
import java.util.Set;
|
|
||||||
|
|
||||||
import javax.servlet.FilterConfig;
|
|
||||||
import javax.servlet.ServletException;
|
|
||||||
import javax.ws.rs.core.HttpHeaders;
|
|
||||||
import javax.ws.rs.core.MediaType;
|
|
||||||
import javax.xml.parsers.DocumentBuilder;
|
|
||||||
import javax.xml.parsers.DocumentBuilderFactory;
|
|
||||||
import javax.xml.parsers.ParserConfigurationException;
|
|
||||||
|
|
||||||
import com.sun.jersey.api.client.config.DefaultClientConfig;
|
import com.sun.jersey.api.client.config.DefaultClientConfig;
|
||||||
|
import com.sun.jersey.api.client.filter.LoggingFilter;
|
||||||
|
import com.sun.jersey.api.json.JSONJAXBContext;
|
||||||
|
import com.sun.jersey.api.json.JSONMarshaller;
|
||||||
|
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
|
||||||
|
import com.sun.jersey.test.framework.WebAppDescriptor;
|
||||||
import org.apache.commons.codec.binary.Base64;
|
import org.apache.commons.codec.binary.Base64;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.io.Text;
|
import org.apache.hadoop.io.Text;
|
||||||
@ -51,11 +40,13 @@
|
|||||||
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
|
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
|
||||||
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
|
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
|
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
|
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResource;
|
import org.apache.hadoop.yarn.api.records.LocalResource;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResourceType;
|
import org.apache.hadoop.yarn.api.records.LocalResourceType;
|
||||||
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
|
||||||
import org.apache.hadoop.yarn.api.records.QueueACL;
|
import org.apache.hadoop.yarn.api.records.QueueACL;
|
||||||
|
import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
|
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
|
||||||
@ -67,12 +58,12 @@
|
|||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState;
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo;
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo;
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.*;
|
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
|
||||||
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
|
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
|
||||||
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
|
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
|
||||||
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
|
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
|
||||||
@ -91,20 +82,36 @@
|
|||||||
import org.xml.sax.InputSource;
|
import org.xml.sax.InputSource;
|
||||||
import org.xml.sax.SAXException;
|
import org.xml.sax.SAXException;
|
||||||
|
|
||||||
import com.google.inject.Guice;
|
import javax.servlet.FilterConfig;
|
||||||
import com.google.inject.Injector;
|
import javax.servlet.ServletException;
|
||||||
import com.google.inject.Singleton;
|
import javax.ws.rs.core.HttpHeaders;
|
||||||
import com.google.inject.servlet.GuiceServletContextListener;
|
import javax.ws.rs.core.MediaType;
|
||||||
import com.google.inject.servlet.ServletModule;
|
import javax.xml.parsers.DocumentBuilder;
|
||||||
import com.sun.jersey.api.client.Client;
|
import javax.xml.parsers.DocumentBuilderFactory;
|
||||||
import com.sun.jersey.api.client.ClientResponse;
|
import javax.xml.parsers.ParserConfigurationException;
|
||||||
import com.sun.jersey.api.client.ClientResponse.Status;
|
import java.io.ByteArrayInputStream;
|
||||||
import com.sun.jersey.api.client.WebResource;
|
import java.io.DataInputStream;
|
||||||
import com.sun.jersey.api.client.filter.LoggingFilter;
|
import java.io.File;
|
||||||
import com.sun.jersey.api.json.JSONJAXBContext;
|
import java.io.FileWriter;
|
||||||
import com.sun.jersey.api.json.JSONMarshaller;
|
import java.io.IOException;
|
||||||
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
|
import java.io.PrintWriter;
|
||||||
import com.sun.jersey.test.framework.WebAppDescriptor;
|
import java.io.StringReader;
|
||||||
|
import java.io.StringWriter;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.Enumeration;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Properties;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertEquals;
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
|
import static org.junit.Assume.assumeTrue;
|
||||||
|
|
||||||
@RunWith(Parameterized.class)
|
@RunWith(Parameterized.class)
|
||||||
public class TestRMWebServicesAppsModification extends JerseyTestBase {
|
public class TestRMWebServicesAppsModification extends JerseyTestBase {
|
||||||
@ -807,7 +814,7 @@ public void testAppSubmit(String acceptMedia, String contentMedia)
|
|||||||
|
|
||||||
RMApp app =
|
RMApp app =
|
||||||
rm.getRMContext().getRMApps()
|
rm.getRMContext().getRMApps()
|
||||||
.get(ConverterUtils.toApplicationId(appId));
|
.get(ApplicationId.fromString(appId));
|
||||||
assertEquals(appName, app.getName());
|
assertEquals(appName, app.getName());
|
||||||
assertEquals(webserviceUserName, app.getUser());
|
assertEquals(webserviceUserName, app.getUser());
|
||||||
assertEquals(2, app.getMaxAppAttempts());
|
assertEquals(2, app.getMaxAppAttempts());
|
||||||
@ -825,8 +832,7 @@ public void testAppSubmit(String acceptMedia, String contentMedia)
|
|||||||
Map<String, LocalResource> appLRs = ctx.getLocalResources();
|
Map<String, LocalResource> appLRs = ctx.getLocalResources();
|
||||||
assertTrue(appLRs.containsKey(lrKey));
|
assertTrue(appLRs.containsKey(lrKey));
|
||||||
LocalResource exampleLR = appLRs.get(lrKey);
|
LocalResource exampleLR = appLRs.get(lrKey);
|
||||||
assertEquals(ConverterUtils.getYarnUrlFromURI(y.getUrl()),
|
assertEquals(URL.fromURI(y.getUrl()), exampleLR.getResource());
|
||||||
exampleLR.getResource());
|
|
||||||
assertEquals(y.getSize(), exampleLR.getSize());
|
assertEquals(y.getSize(), exampleLR.getSize());
|
||||||
assertEquals(y.getTimestamp(), exampleLR.getTimestamp());
|
assertEquals(y.getTimestamp(), exampleLR.getTimestamp());
|
||||||
assertEquals(y.getType(), exampleLR.getType());
|
assertEquals(y.getType(), exampleLR.getType());
|
||||||
|
@ -50,6 +50,7 @@
|
|||||||
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
|
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
|
||||||
import org.apache.hadoop.security.token.Token;
|
import org.apache.hadoop.security.token.Token;
|
||||||
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator;
|
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
|
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
|
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
|
||||||
@ -236,11 +237,11 @@ public void testDelegationTokenAuth() throws Exception {
|
|||||||
|
|
||||||
boolean appExists =
|
boolean appExists =
|
||||||
rm.getRMContext().getRMApps()
|
rm.getRMContext().getRMApps()
|
||||||
.containsKey(ConverterUtils.toApplicationId(appid));
|
.containsKey(ApplicationId.fromString(appid));
|
||||||
assertTrue(appExists);
|
assertTrue(appExists);
|
||||||
RMApp actualApp =
|
RMApp actualApp =
|
||||||
rm.getRMContext().getRMApps()
|
rm.getRMContext().getRMApps()
|
||||||
.get(ConverterUtils.toApplicationId(appid));
|
.get(ApplicationId.fromString(appid));
|
||||||
String owner = actualApp.getUser();
|
String owner = actualApp.getUser();
|
||||||
assertEquals("client", owner);
|
assertEquals("client", owner);
|
||||||
}
|
}
|
||||||
|
@ -37,6 +37,7 @@
|
|||||||
import org.apache.hadoop.minikdc.MiniKdc;
|
import org.apache.hadoop.minikdc.MiniKdc;
|
||||||
import org.apache.hadoop.security.UserGroupInformation;
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
import org.apache.hadoop.security.authentication.KerberosTestUtils;
|
import org.apache.hadoop.security.authentication.KerberosTestUtils;
|
||||||
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
|
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
||||||
@ -236,11 +237,11 @@ private void testAnonymousSimpleUser() throws Exception {
|
|||||||
assertEquals(Status.ACCEPTED.getStatusCode(), conn.getResponseCode());
|
assertEquals(Status.ACCEPTED.getStatusCode(), conn.getResponseCode());
|
||||||
boolean appExists =
|
boolean appExists =
|
||||||
rm.getRMContext().getRMApps()
|
rm.getRMContext().getRMApps()
|
||||||
.containsKey(ConverterUtils.toApplicationId(appid));
|
.containsKey(ApplicationId.fromString(appid));
|
||||||
assertTrue(appExists);
|
assertTrue(appExists);
|
||||||
RMApp actualApp =
|
RMApp actualApp =
|
||||||
rm.getRMContext().getRMApps()
|
rm.getRMContext().getRMApps()
|
||||||
.get(ConverterUtils.toApplicationId(appid));
|
.get(ApplicationId.fromString(appid));
|
||||||
String owner = actualApp.getUser();
|
String owner = actualApp.getUser();
|
||||||
assertEquals(
|
assertEquals(
|
||||||
rm.getConfig().get(CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER,
|
rm.getConfig().get(CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER,
|
||||||
@ -259,11 +260,11 @@ private void testAnonymousSimpleUser() throws Exception {
|
|||||||
conn.getInputStream();
|
conn.getInputStream();
|
||||||
appExists =
|
appExists =
|
||||||
rm.getRMContext().getRMApps()
|
rm.getRMContext().getRMApps()
|
||||||
.containsKey(ConverterUtils.toApplicationId(appid));
|
.containsKey(ApplicationId.fromString(appid));
|
||||||
assertTrue(appExists);
|
assertTrue(appExists);
|
||||||
actualApp =
|
actualApp =
|
||||||
rm.getRMContext().getRMApps()
|
rm.getRMContext().getRMApps()
|
||||||
.get(ConverterUtils.toApplicationId(appid));
|
.get(ApplicationId.fromString(appid));
|
||||||
owner = actualApp.getUser();
|
owner = actualApp.getUser();
|
||||||
assertEquals("client", owner);
|
assertEquals("client", owner);
|
||||||
|
|
||||||
|
@ -481,7 +481,7 @@ private static ApplicationId parseApplicationId(String appIdStr) {
|
|||||||
ApplicationId appId = null;
|
ApplicationId appId = null;
|
||||||
if (appIdStr.startsWith(ApplicationId.appIdStrPrefix)) {
|
if (appIdStr.startsWith(ApplicationId.appIdStrPrefix)) {
|
||||||
try {
|
try {
|
||||||
appId = ConverterUtils.toApplicationId(appIdStr);
|
appId = ApplicationId.fromString(appIdStr);
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
appId = null;
|
appId = null;
|
||||||
}
|
}
|
||||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user