Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-934. Updated patch for HistoryWriter to split the write operations into a startOp and a finishOp. Contributed by Zhijie Shen. 

YARN-947. Updated patch to have separate data object for startOp and finishOp. Contributed by Zhijie Shen.
YARN-956. Updated patch for In-memory Storage after the above two changes.
svn merge --ignore-ancestry -c 1556726 ../YARN-321


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1562182 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2014-01-28 19:50:28 +00:00
parent e4d6b2f485
commit 3ec30a2941
26 changed files with 2429 additions and 574 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationAttemptState.java Normal file
View File

@ -0,0 +1,66 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.api.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
/**
* Enumeration of various states of a <code>RMAppAttempt</code>.
*/
@Public
@Stable
public enum YarnApplicationAttemptState {
/** AppAttempt was just created. */
NEW,
/** AppAttempt has been submitted. */
SUBMITTED,
/**AppAttempt was scheduled*/
SCHEDULED,
/**Acquired AM Container from Scheduler and Saving AppAttempt Data*/
ALLOCATED_SAVING,
/**AppAttempt Data was saved*/
ALLOCATED,
/**AppAttempt was launched*/
LAUNCHED,
/** AppAttempt failed. */
FAILED,
/** AppAttempt is currently running. */
RUNNING,
/** AppAttempt is waiting for state bing saved*/
FINAL_SAVING,
/** AppAttempt is finishing. */
FINISHING,
/** AppAttempt finished successfully. */
FINISHED,
/** AppAttempt was terminated by a user or admin. */
KILLED
}

55
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto
View File

@ -35,6 +35,25 @@ message ApplicationHistoryDataProto {
optional int64 finish_time = 8;
optional string diagnostics_info = 9;
optional FinalApplicationStatusProto final_application_status = 10;
optional YarnApplicationStateProto yarn_application_state = 11;
}
message ApplicationStartDataProto {
optional ApplicationIdProto application_id = 1;
optional string application_name = 2;
optional string application_type = 3;
optional string user = 4;
optional string queue = 5;
optional int64 submit_time = 6;
optional int64 start_time = 7;
}
message ApplicationFinishDataProto {
optional ApplicationIdProto application_id = 1;
optional int64 finish_time = 2;
optional string diagnostics_info = 3;
optional FinalApplicationStatusProto final_application_status = 4;
optional YarnApplicationStateProto yarn_application_state = 5;
}
message ApplicationAttemptHistoryDataProto {
@ -45,6 +64,22 @@ message ApplicationAttemptHistoryDataProto {
optional string diagnostics_info = 5;
optional FinalApplicationStatusProto final_application_status = 6;
optional ContainerIdProto master_container_id = 7;
optional YarnApplicationAttemptStateProto yarn_application_attempt_state = 8;
}
message ApplicationAttemptStartDataProto {
optional ApplicationAttemptIdProto application_attempt_id = 1;
optional string host = 2;
optional int32 rpc_port = 3;
optional ContainerIdProto master_container_id = 4;
}
message ApplicationAttemptFinishDataProto {
optional ApplicationAttemptIdProto application_attempt_id = 1;
optional string tracking_url = 2;
optional string diagnostics_info = 3;
optional FinalApplicationStatusProto final_application_status = 4;
optional YarnApplicationAttemptStateProto yarn_application_attempt_state = 5;
}
message ContainerHistoryDataProto {
@ -56,5 +91,23 @@ message ContainerHistoryDataProto {
optional int64 finish_time = 6;
optional string diagnostics_info = 7;
optional string log_url = 8;
optional ContainerStateProto final_container_status = 9;
optional int32 container_exit_status = 9;
optional ContainerStateProto container_state = 10;
}
message ContainerStartDataProto {
optional ContainerIdProto container_id = 1;
optional ResourceProto allocated_resource = 2;
optional NodeIdProto assigned_node_id = 3;
optional PriorityProto priority = 4;
optional int64 start_time = 5;
}
message ContainerFinishDataProto {
optional ContainerIdProto container_id = 1;
optional int64 finish_time = 2;
optional string diagnostics_info = 3;
optional string log_url = 4;
optional int32 container_exit_status = 5;
optional ContainerStateProto container_state = 6;
}

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
View File

@ -98,6 +98,21 @@ enum YarnApplicationStateProto {
KILLED = 8;
}
enum YarnApplicationAttemptStateProto {
APP_ATTEMPT_NEW = 1;
APP_ATTEMPT_SUBMITTED = 2;
APP_ATTEMPT_SCHEDULED = 3;
APP_ATTEMPT_ALLOCATED_SAVING = 4;
APP_ATTEMPT_ALLOCATED = 5;
APP_ATTEMPT_LAUNCHED = 6;
APP_ATTEMPT_FAILED = 7;
APP_ATTEMPT_RUNNING = 8;
APP_ATTEMPT_FINAL_SAVING = 9;
APP_ATTEMPT_FINISHING = 10;
APP_ATTEMPT_FINISHED = 11;
APP_ATTEMPT_KILLED = 12;
}
enum FinalApplicationStatusProto {
APP_UNDEFINED = 0;
APP_SUCCEEDED = 1;

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
View File

@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.proto.YarnProtos.AMCommandProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAccessTypeProto;
@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueACLProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
import com.google.protobuf.ByteString;
@ -96,6 +98,21 @@ public class ProtoUtils {
return YarnApplicationState.valueOf(e.name());
}
/*
* YarnApplicationAttemptState
*/
private static String YARN_APPLICATION_ATTEMPT_STATE_PREFIX = "APP_ATTEMPT_";
public static YarnApplicationAttemptStateProto convertToProtoFormat(
YarnApplicationAttemptState e) {
return YarnApplicationAttemptStateProto
.valueOf(YARN_APPLICATION_ATTEMPT_STATE_PREFIX + e.name());
}
public static YarnApplicationAttemptState convertFromProtoFormat(
YarnApplicationAttemptStateProto e) {
return YarnApplicationAttemptState.valueOf(e.name().replace(
YARN_APPLICATION_ATTEMPT_STATE_PREFIX, ""));
}
/*
* ApplicationResourceUsageReport
*/

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java
View File

@ -20,9 +20,18 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.service.Service;
/**
* This class is the abstract of the storage of the application history data. It
* is a {@link Service}, such that the implementation of this class can make use
* of the service life cycle to initialize and cleanup the storage. Users can
* access the storage via {@link ApplicationHistoryReader} and
* {@link ApplicationHistoryWriter} interfaces.
*
*/
@InterfaceAudience.Public
@InterfaceStability.Unstable
public interface ApplicationHistoryStore extends ApplicationHistoryReader,
ApplicationHistoryWriter {
public interface ApplicationHistoryStore extends Service,
ApplicationHistoryReader, ApplicationHistoryWriter {
}

84
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryWriter.java
View File

@ -18,11 +18,16 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptStartData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationFinishData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationStartData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerFinishData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerStartData;
/**
* It is the interface of writing the application history, exposing the methods
@ -34,25 +39,72 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.records.Container
public interface ApplicationHistoryWriter {
/**
* This method persists an {@link ApplicationHistoryData} object.
* @param app the {@link ApplicationHistoryData} object
* @throws Throwable
* This method writes the information of <code>RMApp</code> that is available
* when it starts.
*
* @param appStart
* the record of the information of <code>RMApp</code> that is
* available when it starts
* @throws IOException
*/
void writeApplication(ApplicationHistoryData app) throws Throwable;
void applicationStarted(ApplicationStartData appStart) throws IOException;
/**
* This method persists an {@link ApplicationAttemptHistoryData} object.
* @param appAttempt the {@link ApplicationAttemptHistoryData} object
* @throws Throwable
* This method writes the information of <code>RMApp</code> that is available
* when it finishes.
*
* @param appFinish
* the record of the information of <code>RMApp</code> that is
* available when it finishes
* @throws IOException
*/
void writeApplicationAttempt(
ApplicationAttemptHistoryData appAttempt) throws Throwable;
void applicationFinished(ApplicationFinishData appFinish) throws IOException;
/**
* This method persists a {@link ContainerHistoryData} object.
* @param container the {@link ContainerHistoryData} object
* @throws Throwable
* This method writes the information of <code>RMAppAttempt</code> that is
* available when it starts.
*
* @param appAttemptStart
* the record of the information of <code>RMAppAttempt</code> that is
* available when it starts
* @throws IOException
*/
void writeContainer(ContainerHistoryData container) throws Throwable;
void applicationAttemptStarted(
ApplicationAttemptStartData appAttemptStart) throws IOException;
/**
* This method writes the information of <code>RMAppAttempt</code> that is
* available when it finishes.
*
* @param appAttemptFinish
* the record of the information of <code>RMAppAttempt</code> that is
* available when it finishes
* @throws IOException
*/
void applicationAttemptFinished(
ApplicationAttemptFinishData appAttemptFinish) throws IOException;
/**
* This method writes the information of <code>RMContainer</code> that is
* available when it starts.
*
* @param containerStart
* the record of the information of <code>RMContainer</code> that is
* available when it starts
* @throws IOException
*/
void containerStarted(ContainerStartData containerStart) throws IOException;
/**
* This method writes the information of <code>RMContainer</code> that is
* available when it finishes.
*
* @param containerFinish
* the record of the information of <code>RMContainer</code> that is
* available when it finishes
* @throws IOException
*/
void containerFinished(ContainerFinishData containerFinish)
throws IOException;
}

315
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java
View File

@ -19,46 +19,55 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptStartData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationFinishData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationStartData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerFinishData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerStartData;
public class MemoryApplicationHistoryStore implements ApplicationHistoryStore {
/**
* In-memory implementation of {@link ApplicationHistoryStore}.
* This implementation is for test purpose only. If users improperly instantiate
* it, they may encounter reading and writing history data in different memory
* store.
*
*/
@Private
@Unstable
public class MemoryApplicationHistoryStore extends AbstractService
implements ApplicationHistoryStore {
private static MemoryApplicationHistoryStore memStore = null;
private ConcurrentHashMap<ApplicationId, ApplicationHistoryData> applicationData =
private final ConcurrentMap<ApplicationId, ApplicationHistoryData> applicationData =
new ConcurrentHashMap<ApplicationId, ApplicationHistoryData>();
private ConcurrentHashMap<ApplicationId, ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>> applicationAttemptData =
new ConcurrentHashMap<ApplicationId, ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>>();
private ConcurrentHashMap<ContainerId, ContainerHistoryData> containerData =
new ConcurrentHashMap<ContainerId, ContainerHistoryData>();
private final ConcurrentMap<ApplicationId, ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData>> applicationAttemptData =
new ConcurrentHashMap<ApplicationId, ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData>>();
private final ConcurrentMap<ApplicationAttemptId, ConcurrentMap<ContainerId, ContainerHistoryData>> containerData =
new ConcurrentHashMap<ApplicationAttemptId, ConcurrentMap<ContainerId, ContainerHistoryData>>();
private MemoryApplicationHistoryStore() {
}
public static MemoryApplicationHistoryStore getMemoryStore() {
if (memStore == null) {
memStore = new MemoryApplicationHistoryStore();
}
return memStore;
public MemoryApplicationHistoryStore() {
super(MemoryApplicationHistoryStore.class.getName());
}
@Override
public Map<ApplicationId, ApplicationHistoryData> getAllApplications() {
Map<ApplicationId, ApplicationHistoryData> listApps =
new HashMap<ApplicationId, ApplicationHistoryData>();
for (ApplicationId appId : applicationData.keySet()) {
listApps.put(appId, applicationData.get(appId));
}
return listApps;
return new HashMap<ApplicationId, ApplicationHistoryData>(
applicationData);
}
@Override
@ -67,111 +76,209 @@ public class MemoryApplicationHistoryStore implements ApplicationHistoryStore {
}
@Override
public Map<ApplicationAttemptId, ApplicationAttemptHistoryData> getApplicationAttempts(
ApplicationId appId) {
Map<ApplicationAttemptId, ApplicationAttemptHistoryData> listAttempts =
null;
ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttempts =
public Map<ApplicationAttemptId, ApplicationAttemptHistoryData>
getApplicationAttempts(
ApplicationId appId) {
ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData> subMap =
applicationAttemptData.get(appId);
if (appAttempts != null) {
listAttempts =
new HashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>();
for (ApplicationAttemptId attemptId : appAttempts.keySet()) {
listAttempts.put(attemptId, appAttempts.get(attemptId));
}
if (subMap == null) {
return Collections.<ApplicationAttemptId, ApplicationAttemptHistoryData>emptyMap();
} else {
return new HashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>(subMap);
}
return listAttempts;
}
@Override
public ApplicationAttemptHistoryData getApplicationAttempt(
ApplicationAttemptId appAttemptId) {
ApplicationAttemptHistoryData appAttemptHistoryData = null;
ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttempts =
ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData> subMap =
applicationAttemptData.get(appAttemptId.getApplicationId());
if (appAttempts != null) {
appAttemptHistoryData = appAttempts.get(appAttemptId);
if (subMap == null) {
return null;
} else {
return subMap.get(appAttemptId);
}
return appAttemptHistoryData;
}
@Override
public ContainerHistoryData getAMContainer(ApplicationAttemptId appAttemptId) {
ContainerHistoryData Container = null;
ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttempts =
applicationAttemptData.get(appAttemptId.getApplicationId());
if (appAttempts != null) {
containerData.get(appAttempts.get(appAttemptId).getMasterContainerId());
public ContainerHistoryData getAMContainer(
ApplicationAttemptId appAttemptId) {
ApplicationAttemptHistoryData appAttempt =
getApplicationAttempt(appAttemptId);
if (appAttempt == null || appAttempt.getMasterContainerId() == null) {
return null;
} else {
return getContainer(appAttempt.getMasterContainerId());
}
return Container;
}
@Override
public ContainerHistoryData getContainer(ContainerId containerId) {
return containerData.get(containerId);
}
@Override
public void writeApplication(ApplicationHistoryData app) throws Throwable {
if (app != null) {
ApplicationHistoryData oldData =
applicationData.putIfAbsent(app.getApplicationId(), app);
if (oldData != null) {
throw new IOException("This application "
+ app.getApplicationId().toString() + " is already present.");
}
}
}
@Override
public void writeApplicationAttempt(ApplicationAttemptHistoryData appAttempt)
throws Throwable {
if (appAttempt != null) {
if (applicationAttemptData.containsKey(appAttempt
.getApplicationAttemptId().getApplicationId())) {
ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttemptmap =
applicationAttemptData.get(appAttempt.getApplicationAttemptId()
.getApplicationId());
ApplicationAttemptHistoryData oldAppAttempt =
appAttemptmap.putIfAbsent(appAttempt.getApplicationAttemptId(),
appAttempt);
if (oldAppAttempt != null) {
throw new IOException("This application attempt "
+ appAttempt.getApplicationAttemptId().toString()
+ " already present.");
}
} else {
ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttemptmap =
new ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>();
appAttemptmap.put(appAttempt.getApplicationAttemptId(), appAttempt);
applicationAttemptData.putIfAbsent(appAttempt.getApplicationAttemptId()
.getApplicationId(), appAttemptmap);
}
}
}
@Override
public void writeContainer(ContainerHistoryData container) throws Throwable {
if (container != null) {
ContainerHistoryData oldContainer =
containerData.putIfAbsent(container.getContainerId(), container);
if (oldContainer != null) {
throw new IOException("This container "
+ container.getContainerId().toString() + " is already present.");
}
Map<ContainerId, ContainerHistoryData> subMap =
containerData.get(containerId.getApplicationAttemptId());
if (subMap == null) {
return null;
} else {
return subMap.get(containerId);
}
}
@Override
public Map<ContainerId, ContainerHistoryData> getContainers(
ApplicationAttemptId appAttemptId) throws IOException {
HashMap<ContainerId, ContainerHistoryData> containers =
new HashMap<ContainerId, ContainerHistoryData>();
for (ContainerId container : containerData.keySet()) {
if (container.getApplicationAttemptId().equals(appAttemptId)) {
containers.put(container, containerData.get(container));
}
ConcurrentMap<ContainerId, ContainerHistoryData> subMap =
containerData.get(appAttemptId);
if (subMap == null) {
return Collections.<ContainerId, ContainerHistoryData>emptyMap();
} else {
return new HashMap<ContainerId, ContainerHistoryData>(subMap);
}
return containers;
}
@Override
public void applicationStarted(ApplicationStartData appStart)
throws IOException {
ApplicationHistoryData oldData =
applicationData.putIfAbsent(appStart.getApplicationId(),
ApplicationHistoryData.newInstance(
appStart.getApplicationId(),
appStart.getApplicationName(),
appStart.getApplicationType(),
appStart.getQueue(),
appStart.getUser(),
appStart.getSubmitTime(),
appStart.getStartTime(),
Long.MAX_VALUE, null, null, null));
if (oldData != null) {
throw new IOException("The start information of application "
+ appStart.getApplicationId() + " is already stored.");
}
}
@Override
public void applicationFinished(ApplicationFinishData appFinish)
throws IOException {
ApplicationHistoryData data =
applicationData.get(appFinish.getApplicationId());
if (data == null) {
throw new IOException("The finish information of application "
+ appFinish.getApplicationId() + " is stored before the start"
+ " information.");
}
// Make the assumption that YarnApplicationState should not be null if
// the finish information is already recorded
if (data.getYarnApplicationState() != null) {
throw new IOException("The finish information of application "
+ appFinish.getApplicationId() + " is already stored.");
}
data.setFinishTime(appFinish.getFinishTime());
data.setDiagnosticsInfo(appFinish.getDiagnosticsInfo());
data.setFinalApplicationStatus(appFinish.getFinalApplicationStatus());
data.setYarnApplicationState(appFinish.getYarnApplicationState());
}
@Override
public void applicationAttemptStarted(
ApplicationAttemptStartData appAttemptStart) throws IOException {
ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData> subMap =
getSubMap(appAttemptStart.getApplicationAttemptId().getApplicationId());
ApplicationAttemptHistoryData oldData = subMap.putIfAbsent(
appAttemptStart.getApplicationAttemptId(),
ApplicationAttemptHistoryData.newInstance(
appAttemptStart.getApplicationAttemptId(),
appAttemptStart.getHost(),
appAttemptStart.getRPCPort(),
appAttemptStart.getMasterContainerId(),
null, null, null, null));
if (oldData != null) {
throw new IOException("The start information of application attempt "
+ appAttemptStart.getApplicationAttemptId()
+ " is already stored.");
}
}
@Override
public void applicationAttemptFinished(
ApplicationAttemptFinishData appAttemptFinish) throws IOException {
ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData> subMap =
getSubMap(appAttemptFinish.getApplicationAttemptId().getApplicationId());
ApplicationAttemptHistoryData data =
subMap.get(appAttemptFinish.getApplicationAttemptId());
if (data == null) {
throw new IOException("The finish information of application attempt "
+ appAttemptFinish.getApplicationAttemptId() + " is stored before"
+ " the start information.");
}
// Make the assumption that YarnApplicationAttemptState should not be null
// if the finish information is already recorded
if (data.getYarnApplicationAttemptState() != null) {
throw new IOException("The finish information of application attempt "
+ appAttemptFinish.getApplicationAttemptId()
+ " is already stored.");
}
data.setTrackingURL(appAttemptFinish.getTrackingURL());
data.setDiagnosticsInfo(appAttemptFinish.getDiagnosticsInfo());
data.setFinalApplicationStatus(appAttemptFinish.getFinalApplicationStatus());
data.setYarnApplicationAttemptState(appAttemptFinish.getYarnApplicationAttemptState());
}
private ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData>
getSubMap(ApplicationId appId) {
applicationAttemptData.putIfAbsent(appId,
new ConcurrentHashMap<ApplicationAttemptId,
ApplicationAttemptHistoryData>());
return applicationAttemptData.get(appId);
}
@Override
public void containerStarted(ContainerStartData containerStart)
throws IOException {
ConcurrentMap<ContainerId, ContainerHistoryData> subMap =
getSubMap(containerStart.getContainerId().getApplicationAttemptId());
ContainerHistoryData oldData = subMap.putIfAbsent(
containerStart.getContainerId(),
ContainerHistoryData.newInstance(
containerStart.getContainerId(),
containerStart.getAllocatedResource(),
containerStart.getAssignedNode(),
containerStart.getPriority(),
containerStart.getStartTime(),
Long.MAX_VALUE, null, null, Integer.MAX_VALUE, null));
if (oldData != null) {
throw new IOException("The start information of container "
+ containerStart.getContainerId() + " is already stored.");
}
}
@Override
public void containerFinished(ContainerFinishData containerFinish)
throws IOException {
ConcurrentMap<ContainerId, ContainerHistoryData> subMap =
getSubMap(containerFinish.getContainerId().getApplicationAttemptId());
ContainerHistoryData data = subMap.get(containerFinish.getContainerId());
if (data == null) {
throw new IOException("The finish information of container "
+ containerFinish.getContainerId() + " is stored before"
+ " the start information.");
}
// Make the assumption that ContainerState should not be null if
// the finish information is already recorded
if (data.getContainerState() != null) {
throw new IOException("The finish information of container "
+ containerFinish.getContainerId() + " is already stored.");
}
data.setFinishTime(containerFinish.getFinishTime());
data.setDiagnosticsInfo(containerFinish.getDiagnosticsInfo());
data.setLogURL(containerFinish.getLogURL());
data.setContainerExitStatus(containerFinish.getContainerExitStatus());
data.setContainerState(containerFinish.getContainerState());
}
private ConcurrentMap<ContainerId, ContainerHistoryData> getSubMap(
ApplicationAttemptId appAttemptId) {
containerData.putIfAbsent(appAttemptId,
new ConcurrentHashMap<ContainerId, ContainerHistoryData>());
return containerData.get(appAttemptId);
}
}

95
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptFinishData.java Normal file
View File

@ -0,0 +1,95 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.applicationhistoryservice.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.util.Records;
/**
* The class contains the fields that can be determined when
* <code>RMAppAttempt</code> finishes, and that need to be stored persistently.
*/
@Public
@Unstable
public abstract class ApplicationAttemptFinishData {
@Public
@Unstable
public static ApplicationAttemptFinishData newInstance(
ApplicationAttemptId appAttemptId, String diagnosticsInfo,
String trackingURL, FinalApplicationStatus finalApplicationStatus,
YarnApplicationAttemptState yarnApplicationAttemptState) {
ApplicationAttemptFinishData appAttemptFD =
Records.newRecord(ApplicationAttemptFinishData.class);
appAttemptFD.setApplicationAttemptId(appAttemptId);
appAttemptFD.setDiagnosticsInfo(diagnosticsInfo);
appAttemptFD.setTrackingURL(trackingURL);
appAttemptFD.setFinalApplicationStatus(finalApplicationStatus);
appAttemptFD.setYarnApplicationAttemptState(yarnApplicationAttemptState);
return appAttemptFD;
}
@Public
@Unstable
public abstract ApplicationAttemptId getApplicationAttemptId();
@Public
@Unstable
public abstract void setApplicationAttemptId(
ApplicationAttemptId applicationAttemptId);
@Public
@Unstable
public abstract String getTrackingURL();
@Public
@Unstable
public abstract void setTrackingURL(String trackingURL);
@Public
@Unstable
public abstract String getDiagnosticsInfo();
@Public
@Unstable
public abstract void setDiagnosticsInfo(String diagnosticsInfo);
@Public
@Unstable
public abstract FinalApplicationStatus getFinalApplicationStatus();
@Public
@Unstable
public abstract void setFinalApplicationStatus(
FinalApplicationStatus finalApplicationStatus);
@Public
@Unstable
public abstract YarnApplicationAttemptState getYarnApplicationAttemptState();
@Public
@Unstable
public abstract void setYarnApplicationAttemptState(
YarnApplicationAttemptState yarnApplicationAttemptState);
}

113
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java
View File

@ -23,69 +23,150 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
/**
* The class contains all the fields that need to be stored persistently for
* The class contains all the fields that are stored persistently for
* <code>RMAppAttempt</code>.
*/
@Public
@Unstable
public interface ApplicationAttemptHistoryData {
public class ApplicationAttemptHistoryData {
private ApplicationAttemptId applicationAttemptId;
private String host;
private int rpcPort;
private String trackingURL;
private String diagnosticsInfo;
private FinalApplicationStatus finalApplicationStatus;
private ContainerId masterContainerId;
private YarnApplicationAttemptState yarnApplicationAttemptState;
@Public
@Unstable
ApplicationAttemptId getApplicationAttemptId();
public static ApplicationAttemptHistoryData newInstance(
ApplicationAttemptId appAttemptId, String host, int rpcPort,
ContainerId masterContainerId, String diagnosticsInfo,
String trackingURL,
FinalApplicationStatus finalApplicationStatus,
YarnApplicationAttemptState yarnApplicationAttemptState) {
ApplicationAttemptHistoryData appAttemptHD =
new ApplicationAttemptHistoryData();
appAttemptHD.setApplicationAttemptId(appAttemptId);
appAttemptHD.setHost(host);
appAttemptHD.setRPCPort(rpcPort);
appAttemptHD.setMasterContainerId(masterContainerId);
appAttemptHD.setDiagnosticsInfo(diagnosticsInfo);
appAttemptHD.setTrackingURL(trackingURL);
appAttemptHD.setFinalApplicationStatus(finalApplicationStatus);
appAttemptHD.setYarnApplicationAttemptState(yarnApplicationAttemptState);
return appAttemptHD;
}
@Public
@Unstable
void setApplicationAttemptId(ApplicationAttemptId applicationAttemptId);
public ApplicationAttemptId getApplicationAttemptId() {
return applicationAttemptId;
}
@Public
@Unstable
String getHost();
public void setApplicationAttemptId(
ApplicationAttemptId applicationAttemptId) {
this.applicationAttemptId = applicationAttemptId;
}
@Public
@Unstable
void setHost(String host);
public String getHost() {
return host;
}
@Public
@Unstable
int getRPCPort();
public void setHost(String host) {
this.host = host;
}
@Public
@Unstable
void setRPCPort(int rpcPort);
public int getRPCPort() {
return rpcPort;
}
@Public
@Unstable
String getTrackingURL();
public void setRPCPort(int rpcPort) {
this.rpcPort = rpcPort;
}
@Public
@Unstable
void setTrackingURL(String trackingURL);
public String getTrackingURL() {
return trackingURL;
}
@Public
@Unstable
String getDiagnosticsInfo();
public void setTrackingURL(String trackingURL) {
this.trackingURL = trackingURL;
}
@Public
@Unstable
void setDiagnosticsInfo(String diagnosticsInfo);
public String getDiagnosticsInfo() {
return diagnosticsInfo;
}
@Public
@Unstable
FinalApplicationStatus getFinalApplicationStatus();
public void setDiagnosticsInfo(String diagnosticsInfo) {
this.diagnosticsInfo = diagnosticsInfo;
}
@Public
@Unstable
void setFinalApplicationStatus(FinalApplicationStatus finalApplicationStatus);
public FinalApplicationStatus getFinalApplicationStatus() {
return finalApplicationStatus;
}
@Public
@Unstable
ContainerId getMasterContainerId();
public void setFinalApplicationStatus(
FinalApplicationStatus finalApplicationStatus) {
this.finalApplicationStatus = finalApplicationStatus;
}
@Public
@Unstable
void setMasterContainerId(ContainerId masterContainerId);
public ContainerId getMasterContainerId() {
return masterContainerId;
}
@Public
@Unstable
public void setMasterContainerId(ContainerId masterContainerId) {
this.masterContainerId = masterContainerId;
}
@Public
@Unstable
public YarnApplicationAttemptState getYarnApplicationAttemptState() {
return yarnApplicationAttemptState;
}
@Public
@Unstable
public void setYarnApplicationAttemptState(
YarnApplicationAttemptState yarnApplicationAttemptState) {
this.yarnApplicationAttemptState = yarnApplicationAttemptState;
}
}

82
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptStartData.java Normal file
View File

@ -0,0 +1,82 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.applicationhistoryservice.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.util.Records;
/**
* The class contains the fields that can be determined when
* <code>RMAppAttempt</code> starts, and that need to be stored persistently.
*/
@Public
@Unstable
public abstract class ApplicationAttemptStartData {
@Public
@Unstable
public static ApplicationAttemptStartData newInstance(
ApplicationAttemptId appAttemptId, String host, int rpcPort,
ContainerId masterContainerId) {
ApplicationAttemptStartData appAttemptSD =
Records.newRecord(ApplicationAttemptStartData.class);
appAttemptSD.setApplicationAttemptId(appAttemptId);
appAttemptSD.setHost(host);
appAttemptSD.setRPCPort(rpcPort);
appAttemptSD.setMasterContainerId(masterContainerId);
return appAttemptSD;
}
@Public
@Unstable
public abstract ApplicationAttemptId getApplicationAttemptId();
@Public
@Unstable
public abstract void setApplicationAttemptId(
ApplicationAttemptId applicationAttemptId);
@Public
@Unstable
public abstract String getHost();
@Public
@Unstable
public abstract void setHost(String host);
@Public
@Unstable
public abstract int getRPCPort();
@Public
@Unstable
public abstract void setRPCPort(int rpcPort);
@Public
@Unstable
public abstract ContainerId getMasterContainerId();
@Public
@Unstable
public abstract void setMasterContainerId(ContainerId masterContainerId);
}

94
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java Normal file
View File

@ -0,0 +1,94 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.applicationhistoryservice.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.util.Records;
/**
* The class contains the fields that can be determined when
* <code>RMApp</code> finishes, and that need to be stored persistently.
*/
@Public
@Unstable
public abstract class ApplicationFinishData {
@Public
@Unstable
public static ApplicationFinishData newInstance(ApplicationId applicationId,
long finishTime, String diagnosticsInfo,
FinalApplicationStatus finalApplicationStatus,
YarnApplicationState yarnApplicationState) {
ApplicationFinishData appFD =
Records.newRecord(ApplicationFinishData.class);
appFD.setApplicationId(applicationId);
appFD.setFinishTime(finishTime);
appFD.setDiagnosticsInfo(diagnosticsInfo);
appFD.setFinalApplicationStatus(finalApplicationStatus);
appFD.setYarnApplicationState(yarnApplicationState);
return appFD;
}
@Public
@Unstable
public abstract ApplicationId getApplicationId();
@Public
@Unstable
public abstract void setApplicationId(ApplicationId applicationId);
@Public
@Unstable
public abstract long getFinishTime();
@Public
@Unstable
public abstract void setFinishTime(long finishTime);
@Public
@Unstable
public abstract String getDiagnosticsInfo();
@Public
@Unstable
public abstract void setDiagnosticsInfo(String diagnosticsInfo);
@Public
@Unstable
public abstract FinalApplicationStatus getFinalApplicationStatus();
@Public
@Unstable
public abstract void setFinalApplicationStatus(
FinalApplicationStatus finalApplicationStatus);
@Public
@Unstable
public abstract YarnApplicationState getYarnApplicationState();
@Public
@Unstable
public abstract void setYarnApplicationState(
YarnApplicationState yarnApplicationState);
}

143
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java
View File

@ -22,93 +22,192 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
/**
* The class contains all the fields that need to be stored persistently for
* The class contains all the fields that are stored persistently for
* <code>RMApp</code>.
*/
@Public
@Unstable
public interface ApplicationHistoryData {
public class ApplicationHistoryData {
private ApplicationId applicationId;
private String applicationName;
private String applicationType;
private String user;
private String queue;
private long submitTime;
private long startTime;
private long finishTime;
private String diagnosticsInfo;
private FinalApplicationStatus finalApplicationStatus;
private YarnApplicationState yarnApplicationState;
@Public
@Unstable
ApplicationId getApplicationId();
public static ApplicationHistoryData newInstance(ApplicationId applicationId,
String applicationName, String applicationType, String queue,
String user, long submitTime, long startTime, long finishTime,
String diagnosticsInfo, FinalApplicationStatus finalApplicationStatus,
YarnApplicationState yarnApplicationState) {
ApplicationHistoryData appHD = new ApplicationHistoryData();
appHD.setApplicationId(applicationId);
appHD.setApplicationName(applicationName);
appHD.setApplicationType(applicationType);
appHD.setQueue(queue);
appHD.setUser(user);
appHD.setSubmitTime(submitTime);
appHD.setStartTime(startTime);
appHD.setFinishTime(finishTime);
appHD.setDiagnosticsInfo(diagnosticsInfo);
appHD.setFinalApplicationStatus(finalApplicationStatus);
appHD.setYarnApplicationState(yarnApplicationState);
return appHD;
}
@Public
@Unstable
void setApplicationId(ApplicationId applicationId);
public ApplicationId getApplicationId() {
return applicationId;
}
@Public
@Unstable
String getApplicationName();
public void setApplicationId(ApplicationId applicationId) {
this.applicationId = applicationId;
}
@Public
@Unstable
void setApplicationName(String applicationName);
public String getApplicationName() {
return applicationName;
}
@Public
@Unstable
String getApplicationType();
public void setApplicationName(String applicationName) {
this.applicationName = applicationName;
}
@Public
@Unstable
void setApplicationType(String applicationType);
public String getApplicationType() {
return applicationType;
}
@Public
@Unstable
String getUser();
public void setApplicationType(String applicationType) {
this.applicationType = applicationType;
}
@Public
@Unstable
void setUser(String user);
public String getUser() {
return user;
}
@Public
@Unstable
String getQueue();
public void setUser(String user) {
this.user = user;
}
@Public
@Unstable
void setQueue(String queue);
public String getQueue() {
return queue;
}
@Public
@Unstable
long getSubmitTime();
public void setQueue(String queue) {
this.queue = queue;
}
@Public
@Unstable
void setSubmitTime(long submitTime);
public long getSubmitTime() {
return submitTime;
}
@Public
@Unstable
long getStartTime();
public void setSubmitTime(long submitTime) {
this.submitTime = submitTime;
}
@Public
@Unstable
void setStartTime(long startTime);
public long getStartTime() {
return startTime;
}
@Public
@Unstable
long getFinishTime();
public void setStartTime(long startTime) {
this.startTime = startTime;
}
@Public
@Unstable
void setFinishTime(long finishTime);
public long getFinishTime() {
return finishTime;
}
@Public
@Unstable
String getDiagnosticsInfo();
public void setFinishTime(long finishTime) {
this.finishTime = finishTime;
}
@Public
@Unstable
void setDiagnosticsInfo(String diagnosticInfo);
public String getDiagnosticsInfo() {
return diagnosticsInfo;
}
@Public
@Unstable
FinalApplicationStatus getFinalApplicationStatus();
public void setDiagnosticsInfo(String diagnosticsInfo) {
this.diagnosticsInfo = diagnosticsInfo;
}
@Public
@Unstable
void setFinalApplicationStatus(FinalApplicationStatus finalApplicationStatus);
public FinalApplicationStatus getFinalApplicationStatus() {
return finalApplicationStatus;
}
@Public
@Unstable
public void setFinalApplicationStatus(
FinalApplicationStatus finalApplicationStatus) {
this.finalApplicationStatus = finalApplicationStatus;
}
@Public
@Unstable
public YarnApplicationState getYarnApplicationState() {
return this.yarnApplicationState;
}
@Public
@Unstable
public void setYarnApplicationState(
YarnApplicationState yarnApplicationState) {
this.yarnApplicationState = yarnApplicationState;
}
}

108
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java Normal file
View File

@ -0,0 +1,108 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.applicationhistoryservice.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.util.Records;
/**
* The class contains the fields that can be determined when
* <code>RMApp</code> starts, and that need to be stored persistently.
*/
@Public
@Unstable
public abstract class ApplicationStartData {
@Public
@Unstable
public static ApplicationStartData newInstance(
ApplicationId applicationId, String applicationName,
String applicationType, String queue, String user,
long submitTime, long startTime) {
ApplicationStartData appSD =
Records.newRecord(ApplicationStartData.class);
appSD.setApplicationId(applicationId);
appSD.setApplicationName(applicationName);
appSD.setApplicationType(applicationType);
appSD.setQueue(queue);
appSD.setUser(user);
appSD.setSubmitTime(submitTime);
appSD.setStartTime(startTime);
return appSD;
}
@Public
@Unstable
public abstract ApplicationId getApplicationId();
@Public
@Unstable
public abstract void setApplicationId(ApplicationId applicationId);
@Public
@Unstable
public abstract String getApplicationName();
@Public
@Unstable
public abstract void setApplicationName(String applicationName);
@Public
@Unstable
public abstract String getApplicationType();
@Public
@Unstable
public abstract void setApplicationType(String applicationType);
@Public
@Unstable
public abstract String getUser();
@Public
@Unstable
public abstract void setUser(String user);
@Public
@Unstable
public abstract String getQueue();
@Public
@Unstable
public abstract void setQueue(String queue);
@Public
@Unstable
public abstract long getSubmitTime();
@Public
@Unstable
public abstract void setSubmitTime(long submitTime);
@Public
@Unstable
public abstract long getStartTime();
@Public
@Unstable
public abstract void setStartTime(long startTime);
}

99
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerFinishData.java Normal file
View File

@ -0,0 +1,99 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.applicationhistoryservice.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.util.Records;
/**
* The class contains the fields that can be determined when
* <code>RMContainer</code> finishes, and that need to be stored persistently.
*/
@Public
@Unstable
public abstract class ContainerFinishData {
@Public
@Unstable
public static ContainerFinishData newInstance(ContainerId containerId,
long finishTime, String diagnosticsInfo, String logURL,
int containerExitCode, ContainerState containerState) {
ContainerFinishData containerFD =
Records.newRecord(ContainerFinishData.class);
containerFD.setContainerId(containerId);
containerFD.setFinishTime(finishTime);
containerFD.setDiagnosticsInfo(diagnosticsInfo);
containerFD.setLogURL(logURL);
containerFD.setContainerExitStatus(containerExitCode);
containerFD.setContainerState(containerState);
return containerFD;
}
@Public
@Unstable
public abstract ContainerId getContainerId();
@Public
@Unstable
public abstract void setContainerId(ContainerId containerId);
@Public
@Unstable
public abstract long getFinishTime();
@Public
@Unstable
public abstract void setFinishTime(long finishTime);
@Public
@Unstable
public abstract String getDiagnosticsInfo();
@Public
@Unstable
public abstract void setDiagnosticsInfo(String diagnosticsInfo);
@Public
@Unstable
public abstract String getLogURL();
@Public
@Unstable
public abstract void setLogURL(String logURL);
@Public
@Unstable
public abstract int getContainerExitStatus();
@Public
@Unstable
public abstract void setContainerExitStatus(int containerExitStatus);
@Public
@Unstable
public abstract ContainerState getContainerState();
@Public
@Unstable
public abstract void setContainerState(ContainerState containerState);
}

128
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java
View File

@ -27,83 +27,171 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
/**
* The class contains all the fields that need to be stored persistently for
* The class contains all the fields that are stored persistently for
* <code>RMContainer</code>.
*/
@Public
@Unstable
public interface ContainerHistoryData {
public class ContainerHistoryData {
private ContainerId containerId;
private Resource allocatedResource;
private NodeId assignedNode;
private Priority priority;
private long startTime;
private long finishTime;
private String diagnosticsInfo;
private String logURL;
private int containerExitStatus;
private ContainerState containerState;
@Public
@Unstable
ContainerId getContainerId();
public static ContainerHistoryData newInstance(ContainerId containerId,
Resource allocatedResource, NodeId assignedNode, Priority priority,
long startTime, long finishTime, String diagnosticsInfo, String logURL,
int containerExitCode, ContainerState containerState) {
ContainerHistoryData containerHD = new ContainerHistoryData();
containerHD.setContainerId(containerId);
containerHD.setAllocatedResource(allocatedResource);
containerHD.setAssignedNode(assignedNode);
containerHD.setPriority(priority);
containerHD.setStartTime(startTime);
containerHD.setFinishTime(finishTime);
containerHD.setDiagnosticsInfo(diagnosticsInfo);
containerHD.setLogURL(logURL);
containerHD.setContainerExitStatus(containerExitCode);
containerHD.setContainerState(containerState);
return containerHD;
}
@Public
@Unstable
void setContainerId(ContainerId containerId);
public ContainerId getContainerId() {
return containerId;
}
@Public
@Unstable
Resource getAllocatedResource();
public void setContainerId(ContainerId containerId) {
this.containerId = containerId;
}
@Public
@Unstable
void setAllocatedResource(Resource resource);
public Resource getAllocatedResource() {
return allocatedResource;
}
@Public
@Unstable
NodeId getAssignedNode();
public void setAllocatedResource(Resource resource) {
this.allocatedResource = resource;
}
@Public
@Unstable
void setAssignedNode(NodeId nodeId);
public NodeId getAssignedNode() {
return assignedNode;
}
@Public
@Unstable
Priority getPriority();
public void setAssignedNode(NodeId nodeId) {
this.assignedNode = nodeId;
}
@Public
@Unstable
void setPriority(Priority priority);
public Priority getPriority() {
return priority;
}
@Public
@Unstable
long getStartTime();
public void setPriority(Priority priority) {
this.priority = priority;
}
@Public
@Unstable
void setStartTime(long startTime);
public long getStartTime() {
return startTime;
}
@Public
@Unstable
long getFinishTime();
public void setStartTime(long startTime) {
this.startTime = startTime;
}
@Public
@Unstable
void setFinishTime(long finishTime);
public long getFinishTime() {
return finishTime;
}
@Public
@Unstable
String getDiagnosticsInfo();
public void setFinishTime(long finishTime) {
this.finishTime = finishTime;
}
@Public
@Unstable
void setDiagnosticsInfo(String diagnosticInfo);
public String getDiagnosticsInfo() {
return diagnosticsInfo;
}
@Public
@Unstable
String getLogURL();
public void setDiagnosticsInfo(String diagnosticsInfo) {
this.diagnosticsInfo = diagnosticsInfo;
}
@Public
@Unstable
void setLogURL(String logURL);
public String getLogURL() {
return logURL;
}
@Public
@Unstable
ContainerState getFinalContainerStatus();
public void setLogURL(String logURL) {
this.logURL = logURL;
}
@Public
@Unstable
void setFinalContainerStatus(ContainerState finalContainerState);
public int getContainerExitStatus() {
return containerExitStatus;
}
@Public
@Unstable
public void setContainerExitStatus(int containerExitStatus) {
this.containerExitStatus = containerExitStatus;
}
@Public
@Unstable
public ContainerState getContainerState() {
return containerState;
}
@Public
@Unstable
public void setContainerState(ContainerState containerState) {
this.containerState = containerState;
}
}

92
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerStartData.java Normal file
View File

@ -0,0 +1,92 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.applicationhistoryservice.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.Records;
/**
* The class contains the fields that can be determined when
* <code>RMContainer</code> starts, and that need to be stored persistently.
*/
@Public
@Unstable
public abstract class ContainerStartData {
@Public
@Unstable
public static ContainerStartData newInstance(ContainerId containerId,
Resource allocatedResource, NodeId assignedNode, Priority priority,
long startTime) {
ContainerStartData containerSD =
Records.newRecord(ContainerStartData.class);
containerSD.setContainerId(containerId);
containerSD.setAllocatedResource(allocatedResource);
containerSD.setAssignedNode(assignedNode);
containerSD.setPriority(priority);
containerSD.setStartTime(startTime);
return containerSD;
}
@Public
@Unstable
public abstract ContainerId getContainerId();
@Public
@Unstable
public abstract void setContainerId(ContainerId containerId);
@Public
@Unstable
public abstract Resource getAllocatedResource();
@Public
@Unstable
public abstract void setAllocatedResource(Resource resource);
@Public
@Unstable
public abstract NodeId getAssignedNode();
@Public
@Unstable
public abstract void setAssignedNode(NodeId nodeId);
@Public
@Unstable
public abstract Priority getPriority();
@Public
@Unstable
public abstract void setPriority(Priority priority);
@Public
@Unstable
public abstract long getStartTime();
@Public
@Unstable
public abstract void setStartTime(long startTime);
}

119
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptHistoryDataPBImpl.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java
View File

@ -19,48 +19,45 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptHistoryDataProto;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptHistoryDataProtoOrBuilder;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptFinishDataProto;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptFinishDataProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData;
import com.google.protobuf.TextFormat;
public class ApplicationAttemptHistoryDataPBImpl
extends ProtoBase<ApplicationAttemptHistoryDataProto>
implements ApplicationAttemptHistoryData {
public class ApplicationAttemptFinishDataPBImpl
extends ApplicationAttemptFinishData {
ApplicationAttemptHistoryDataProto proto =
ApplicationAttemptHistoryDataProto.getDefaultInstance();
ApplicationAttemptHistoryDataProto.Builder builder = null;
ApplicationAttemptFinishDataProto proto =
ApplicationAttemptFinishDataProto.getDefaultInstance();
ApplicationAttemptFinishDataProto.Builder builder = null;
boolean viaProto = false;
public ApplicationAttemptHistoryDataPBImpl() {
builder = ApplicationAttemptHistoryDataProto.newBuilder();
public ApplicationAttemptFinishDataPBImpl() {
builder = ApplicationAttemptFinishDataProto.newBuilder();
}
public ApplicationAttemptHistoryDataPBImpl(
ApplicationAttemptHistoryDataProto proto) {
public ApplicationAttemptFinishDataPBImpl(
ApplicationAttemptFinishDataProto proto) {
this.proto = proto;
viaProto = true;
}
private ApplicationAttemptId applicationAttemptId;
private ContainerId masterContainerId;
@Override
public ApplicationAttemptId getApplicationAttemptId() {
if (this.applicationAttemptId != null) {
return this.applicationAttemptId;
}
ApplicationAttemptHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasApplicationAttemptId()) {
return null;
}
@ -79,40 +76,9 @@ public class ApplicationAttemptHistoryDataPBImpl
this.applicationAttemptId = applicationAttemptId;
}
@Override
public String getHost() {
ApplicationAttemptHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasHost()) {
return null;
}
return p.getHost();
}
@Override
public void setHost(String host) {
maybeInitBuilder();
if (host == null) {
builder.clearHost();
return;
}
builder.setHost(host);
}
@Override
public int getRPCPort() {
ApplicationAttemptHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
return p.getRpcPort();
}
@Override
public void setRPCPort(int rpcPort) {
maybeInitBuilder();
builder.setRpcPort(rpcPort);
}
@Override
public String getTrackingURL() {
ApplicationAttemptHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasTrackingUrl()) {
return null;
}
@ -131,7 +97,7 @@ public class ApplicationAttemptHistoryDataPBImpl
@Override
public String getDiagnosticsInfo() {
ApplicationAttemptHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasDiagnosticsInfo()) {
return null;
}
@ -150,7 +116,7 @@ public class ApplicationAttemptHistoryDataPBImpl
@Override
public FinalApplicationStatus getFinalApplicationStatus() {
ApplicationAttemptHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasFinalApplicationStatus()) {
return null;
}
@ -170,30 +136,25 @@ public class ApplicationAttemptHistoryDataPBImpl
}
@Override
public ContainerId getMasterContainerId() {
if (this.masterContainerId != null) {
return this.masterContainerId;
}
ApplicationAttemptHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasApplicationAttemptId()) {
public YarnApplicationAttemptState getYarnApplicationAttemptState() {
ApplicationAttemptFinishDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasYarnApplicationAttemptState()) {
return null;
}
this.masterContainerId =
convertFromProtoFormat(p.getMasterContainerId());
return this.masterContainerId;
return convertFromProtoFormat(p.getYarnApplicationAttemptState());
}
@Override
public void setMasterContainerId(ContainerId masterContainerId) {
public void setYarnApplicationAttemptState(YarnApplicationAttemptState state) {
maybeInitBuilder();
if (masterContainerId == null) {
builder.clearMasterContainerId();
if (state == null) {
builder.clearYarnApplicationAttemptState();
return;
}
this.masterContainerId = masterContainerId;
builder.setYarnApplicationAttemptState(convertToProtoFormat(state));
}
@Override
public ApplicationAttemptHistoryDataProto getProto() {
public ApplicationAttemptFinishDataProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
@ -217,7 +178,7 @@ public class ApplicationAttemptHistoryDataPBImpl
@Override
public String toString() {
return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " ");
return TextFormat.shortDebugString(getProto());
}
private void mergeLocalToBuilder() {
@ -227,12 +188,6 @@ public class ApplicationAttemptHistoryDataPBImpl
builder.setApplicationAttemptId(
convertToProtoFormat(this.applicationAttemptId));
}
if (this.masterContainerId != null && !((ContainerIdPBImpl)
this.masterContainerId).getProto().equals(
builder.getMasterContainerId())) {
builder.setMasterContainerId(
convertToProtoFormat(this.masterContainerId));
}
}
private void mergeLocalToProto() {
@ -246,7 +201,7 @@ public class ApplicationAttemptHistoryDataPBImpl
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ApplicationAttemptHistoryDataProto.newBuilder(proto);
builder = ApplicationAttemptFinishDataProto.newBuilder(proto);
}
viaProto = false;
}
@ -271,14 +226,14 @@ public class ApplicationAttemptHistoryDataPBImpl
return ProtoUtils.convertToProtoFormat(finalApplicationStatus);
}
private ContainerIdPBImpl convertFromProtoFormat(
ContainerIdProto containerId) {
return new ContainerIdPBImpl(containerId);
private YarnApplicationAttemptStateProto convertToProtoFormat(
YarnApplicationAttemptState state) {
return ProtoUtils.convertToProtoFormat(state);
}
private ContainerIdProto convertToProtoFormat(
ContainerId masterContainerId) {
return ((ContainerIdPBImpl) masterContainerId).getProto();
private YarnApplicationAttemptState convertFromProtoFormat(
YarnApplicationAttemptStateProto yarnApplicationAttemptState) {
return ProtoUtils.convertFromProtoFormat(yarnApplicationAttemptState);
}
}

211
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptStartDataPBImpl.java Normal file
View File

@ -0,0 +1,211 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptStartDataProto;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationAttemptStartDataProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptStartData;
import com.google.protobuf.TextFormat;
public class ApplicationAttemptStartDataPBImpl
extends ApplicationAttemptStartData {
ApplicationAttemptStartDataProto proto =
ApplicationAttemptStartDataProto.getDefaultInstance();
ApplicationAttemptStartDataProto.Builder builder = null;
boolean viaProto = false;
public ApplicationAttemptStartDataPBImpl() {
builder = ApplicationAttemptStartDataProto.newBuilder();
}
public ApplicationAttemptStartDataPBImpl(
ApplicationAttemptStartDataProto proto) {
this.proto = proto;
viaProto = true;
}
private ApplicationAttemptId applicationAttemptId;
private ContainerId masterContainerId;
@Override
public ApplicationAttemptId getApplicationAttemptId() {
if (this.applicationAttemptId != null) {
return this.applicationAttemptId;
}
ApplicationAttemptStartDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasApplicationAttemptId()) {
return null;
}
this.applicationAttemptId =
convertFromProtoFormat(p.getApplicationAttemptId());
return this.applicationAttemptId;
}
@Override
public void
setApplicationAttemptId(ApplicationAttemptId applicationAttemptId) {
maybeInitBuilder();
if (applicationAttemptId == null) {
builder.clearApplicationAttemptId();
}
this.applicationAttemptId = applicationAttemptId;
}
@Override
public String getHost() {
ApplicationAttemptStartDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasHost()) {
return null;
}
return p.getHost();
}
@Override
public void setHost(String host) {
maybeInitBuilder();
if (host == null) {
builder.clearHost();
return;
}
builder.setHost(host);
}
@Override
public int getRPCPort() {
ApplicationAttemptStartDataProtoOrBuilder p = viaProto ? proto : builder;
return p.getRpcPort();
}
@Override
public void setRPCPort(int rpcPort) {
maybeInitBuilder();
builder.setRpcPort(rpcPort);
}
@Override
public ContainerId getMasterContainerId() {
if (this.masterContainerId != null) {
return this.masterContainerId;
}
ApplicationAttemptStartDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasApplicationAttemptId()) {
return null;
}
this.masterContainerId =
convertFromProtoFormat(p.getMasterContainerId());
return this.masterContainerId;
}
@Override
public void setMasterContainerId(ContainerId masterContainerId) {
maybeInitBuilder();
if (masterContainerId == null) {
builder.clearMasterContainerId();
}
this.masterContainerId = masterContainerId;
}
public ApplicationAttemptStartDataProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
@Override
public int hashCode() {
return getProto().hashCode();
}
@Override
public boolean equals(Object other) {
if (other == null)
return false;
if (other.getClass().isAssignableFrom(this.getClass())) {
return this.getProto().equals(this.getClass().cast(other).getProto());
}
return false;
}
@Override
public String toString() {
return TextFormat.shortDebugString(getProto());
}
private void mergeLocalToBuilder() {
if (this.applicationAttemptId != null && !((ApplicationAttemptIdPBImpl)
this.applicationAttemptId).getProto().equals(
builder.getApplicationAttemptId())) {
builder.setApplicationAttemptId(
convertToProtoFormat(this.applicationAttemptId));
}
if (this.masterContainerId != null && !((ContainerIdPBImpl)
this.masterContainerId).getProto().equals(
builder.getMasterContainerId())) {
builder.setMasterContainerId(
convertToProtoFormat(this.masterContainerId));
}
}
private void mergeLocalToProto() {
if (viaProto) {
maybeInitBuilder();
}
mergeLocalToBuilder();
proto = builder.build();
viaProto = true;
}
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ApplicationAttemptStartDataProto.newBuilder(proto);
}
viaProto = false;
}
private ApplicationAttemptIdPBImpl convertFromProtoFormat(
ApplicationAttemptIdProto applicationAttemptId) {
return new ApplicationAttemptIdPBImpl(applicationAttemptId);
}
private ApplicationAttemptIdProto convertToProtoFormat(
ApplicationAttemptId applicationAttemptId) {
return ((ApplicationAttemptIdPBImpl) applicationAttemptId).getProto();
}
private ContainerIdPBImpl convertFromProtoFormat(
ContainerIdProto containerId) {
return new ContainerIdPBImpl(containerId);
}
private ContainerIdProto convertToProtoFormat(
ContainerId masterContainerId) {
return ((ContainerIdPBImpl) masterContainerId).getProto();
}
}

228
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java Normal file
View File

@ -0,0 +1,228 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationFinishDataProto;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationFinishDataProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationFinishData;
import com.google.protobuf.TextFormat;
public class ApplicationFinishDataPBImpl
extends ApplicationFinishData {
ApplicationFinishDataProto proto =
ApplicationFinishDataProto.getDefaultInstance();
ApplicationFinishDataProto.Builder builder = null;
boolean viaProto = false;
private ApplicationId applicationId;
public ApplicationFinishDataPBImpl() {
builder = ApplicationFinishDataProto.newBuilder();
}
public ApplicationFinishDataPBImpl(ApplicationFinishDataProto proto) {
this.proto = proto;
viaProto = true;
}
@Override
public ApplicationId getApplicationId() {
if (this.applicationId != null) {
return this.applicationId;
}
ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasApplicationId()) {
return null;
}
this.applicationId = convertFromProtoFormat(p.getApplicationId());
return this.applicationId;
}
@Override
public void setApplicationId(ApplicationId applicationId) {
maybeInitBuilder();
if (applicationId == null) {
builder.clearApplicationId();
}
this.applicationId = applicationId;
}
@Override
public long getFinishTime() {
ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder;
return p.getFinishTime();
}
@Override
public void setFinishTime(long finishTime) {
maybeInitBuilder();
builder.setFinishTime(finishTime);
}
@Override
public String getDiagnosticsInfo() {
ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasDiagnosticsInfo()) {
return null;
}
return p.getDiagnosticsInfo();
}
@Override
public void setDiagnosticsInfo(String diagnosticsInfo) {
maybeInitBuilder();
if (diagnosticsInfo == null) {
builder.clearDiagnosticsInfo();
return;
}
builder.setDiagnosticsInfo(diagnosticsInfo);
}
@Override
public FinalApplicationStatus getFinalApplicationStatus() {
ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasFinalApplicationStatus()) {
return null;
}
return convertFromProtoFormat(p.getFinalApplicationStatus());
}
@Override
public void setFinalApplicationStatus(
FinalApplicationStatus finalApplicationStatus) {
maybeInitBuilder();
if (finalApplicationStatus == null) {
builder.clearFinalApplicationStatus();
return;
}
builder.setFinalApplicationStatus(
convertToProtoFormat(finalApplicationStatus));
}
@Override
public YarnApplicationState getYarnApplicationState() {
ApplicationFinishDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasYarnApplicationState()) {
return null;
}
return convertFromProtoFormat(p.getYarnApplicationState());
}
@Override
public void setYarnApplicationState(YarnApplicationState state) {
maybeInitBuilder();
if (state == null) {
builder.clearYarnApplicationState();
return;
}
builder.setYarnApplicationState(convertToProtoFormat(state));
}
public ApplicationFinishDataProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
@Override
public int hashCode() {
return getProto().hashCode();
}
@Override
public boolean equals(Object other) {
if (other == null)
return false;
if (other.getClass().isAssignableFrom(this.getClass())) {
return this.getProto().equals(this.getClass().cast(other).getProto());
}
return false;
}
@Override
public String toString() {
return TextFormat.shortDebugString(getProto());
}
private void mergeLocalToBuilder() {
if (this.applicationId != null && !((ApplicationIdPBImpl)
this.applicationId).getProto().equals(builder.getApplicationId())) {
builder.setApplicationId(convertToProtoFormat(this.applicationId));
}
}
private void mergeLocalToProto() {
if (viaProto) {
maybeInitBuilder();
}
mergeLocalToBuilder();
proto = builder.build();
viaProto = true;
}
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ApplicationFinishDataProto.newBuilder(proto);
}
viaProto = false;
}
private ApplicationIdProto convertToProtoFormat(
ApplicationId applicationId) {
return ((ApplicationIdPBImpl) applicationId).getProto();
}
private ApplicationIdPBImpl convertFromProtoFormat(
ApplicationIdProto applicationId) {
return new ApplicationIdPBImpl(applicationId);
}
private FinalApplicationStatus convertFromProtoFormat(
FinalApplicationStatusProto finalApplicationStatus) {
return ProtoUtils.convertFromProtoFormat(finalApplicationStatus);
}
private FinalApplicationStatusProto convertToProtoFormat(
FinalApplicationStatus finalApplicationStatus) {
return ProtoUtils.convertToProtoFormat(finalApplicationStatus);
}
private YarnApplicationStateProto convertToProtoFormat(
YarnApplicationState state) {
return ProtoUtils.convertToProtoFormat(state);
}
private YarnApplicationState convertFromProtoFormat(
YarnApplicationStateProto yarnApplicationState) {
return ProtoUtils.convertFromProtoFormat(yarnApplicationState);
}
}

113
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationHistoryDataPBImpl.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java
View File

@ -19,33 +19,30 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationHistoryDataProto;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationHistoryDataProtoOrBuilder;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationStartDataProto;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ApplicationStartDataProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationStartData;
import com.google.protobuf.TextFormat;
public class ApplicationHistoryDataPBImpl
extends ProtoBase<ApplicationHistoryDataProto>
implements ApplicationHistoryData {
public class ApplicationStartDataPBImpl
extends ApplicationStartData {
ApplicationHistoryDataProto proto =
ApplicationHistoryDataProto.getDefaultInstance();
ApplicationHistoryDataProto.Builder builder = null;
ApplicationStartDataProto proto =
ApplicationStartDataProto.getDefaultInstance();
ApplicationStartDataProto.Builder builder = null;
boolean viaProto = false;
private ApplicationId applicationId;
public ApplicationHistoryDataPBImpl() {
builder = ApplicationHistoryDataProto.newBuilder();
public ApplicationStartDataPBImpl() {
builder = ApplicationStartDataProto.newBuilder();
}
public ApplicationHistoryDataPBImpl(ApplicationHistoryDataProto proto) {
public ApplicationStartDataPBImpl(ApplicationStartDataProto proto) {
this.proto = proto;
viaProto = true;
}
@ -55,7 +52,7 @@ public class ApplicationHistoryDataPBImpl
if (this.applicationId != null) {
return this.applicationId;
}
ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasApplicationId()) {
return null;
}
@ -74,7 +71,7 @@ public class ApplicationHistoryDataPBImpl
@Override
public String getApplicationName() {
ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasApplicationName()) {
return null;
}
@ -93,7 +90,7 @@ public class ApplicationHistoryDataPBImpl
@Override
public String getApplicationType() {
ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasApplicationType()) {
return null;
}
@ -112,7 +109,7 @@ public class ApplicationHistoryDataPBImpl
@Override
public String getUser() {
ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasUser()) {
return null;
}
@ -131,7 +128,7 @@ public class ApplicationHistoryDataPBImpl
@Override
public String getQueue() {
ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasQueue()) {
return null;
}
@ -150,7 +147,7 @@ public class ApplicationHistoryDataPBImpl
@Override
public long getSubmitTime() {
ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
return p.getStartTime();
}
@ -162,7 +159,7 @@ public class ApplicationHistoryDataPBImpl
@Override
public long getStartTime() {
ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
ApplicationStartDataProtoOrBuilder p = viaProto ? proto : builder;
return p.getStartTime();
}
@ -172,60 +169,7 @@ public class ApplicationHistoryDataPBImpl
builder.setStartTime(startTime);
}
@Override
public long getFinishTime() {
ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
return p.getFinishTime();
}
@Override
public void setFinishTime(long finishTime) {
maybeInitBuilder();
builder.setFinishTime(finishTime);
}
@Override
public String getDiagnosticsInfo() {
ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasDiagnosticsInfo()) {
return null;
}
return p.getDiagnosticsInfo();
}
@Override
public void setDiagnosticsInfo(String diagnosticInfo) {
maybeInitBuilder();
if (diagnosticInfo == null) {
builder.clearDiagnosticsInfo();
return;
}
builder.setDiagnosticsInfo(diagnosticInfo);
}
@Override
public FinalApplicationStatus getFinalApplicationStatus() {
ApplicationHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasFinalApplicationStatus()) {
return null;
}
return convertFromProtoFormat(p.getFinalApplicationStatus());
}
@Override
public void setFinalApplicationStatus(
FinalApplicationStatus finalApplicationStatus) {
maybeInitBuilder();
if (finalApplicationStatus == null) {
builder.clearFinalApplicationStatus();
return;
}
builder.setFinalApplicationStatus(
convertToProtoFormat(finalApplicationStatus));
}
@Override
public ApplicationHistoryDataProto getProto() {
public ApplicationStartDataProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
@ -249,7 +193,7 @@ public class ApplicationHistoryDataPBImpl
@Override
public String toString() {
return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " ");
return TextFormat.shortDebugString(getProto());
}
private void mergeLocalToBuilder() {
@ -270,7 +214,7 @@ public class ApplicationHistoryDataPBImpl
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ApplicationHistoryDataProto.newBuilder(proto);
builder = ApplicationStartDataProto.newBuilder(proto);
}
viaProto = false;
}
@ -284,15 +228,4 @@ public class ApplicationHistoryDataPBImpl
ApplicationIdProto applicationId) {
return new ApplicationIdPBImpl(applicationId);
}
private FinalApplicationStatus convertFromProtoFormat(
FinalApplicationStatusProto finalApplicationStatus) {
return ProtoUtils.convertFromProtoFormat(finalApplicationStatus);
}
private FinalApplicationStatusProto convertToProtoFormat(
FinalApplicationStatus finalApplicationStatus) {
return ProtoUtils.convertToProtoFormat(finalApplicationStatus);
}
}

226
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerFinishDataPBImpl.java Normal file
View File

@ -0,0 +1,226 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerFinishDataProto;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerFinishDataProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerFinishData;
import com.google.protobuf.TextFormat;
public class ContainerFinishDataPBImpl
extends ContainerFinishData {
ContainerFinishDataProto proto =
ContainerFinishDataProto.getDefaultInstance();
ContainerFinishDataProto.Builder builder = null;
boolean viaProto = false;
private ContainerId containerId;
public ContainerFinishDataPBImpl() {
builder = ContainerFinishDataProto.newBuilder();
}
public ContainerFinishDataPBImpl(ContainerFinishDataProto proto) {
this.proto = proto;
viaProto = true;
}
@Override
public ContainerId getContainerId() {
if (this.containerId != null) {
return this.containerId;
}
ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasContainerId()) {
return null;
}
this.containerId = convertFromProtoFormat(p.getContainerId());
return this.containerId;
}
@Override
public void setContainerId(ContainerId containerId) {
maybeInitBuilder();
if (containerId == null) {
builder.clearContainerId();
}
this.containerId = containerId;
}
@Override
public long getFinishTime() {
ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;
return p.getFinishTime();
}
@Override
public void setFinishTime(long finishTime) {
maybeInitBuilder();
builder.setFinishTime(finishTime);
}
@Override
public String getDiagnosticsInfo() {
ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasDiagnosticsInfo()) {
return null;
}
return p.getDiagnosticsInfo();
}
@Override
public void setDiagnosticsInfo(String diagnosticsInfo) {
maybeInitBuilder();
if (diagnosticsInfo == null) {
builder.clearDiagnosticsInfo();
return;
}
builder.setDiagnosticsInfo(diagnosticsInfo);
}
@Override
public String getLogURL() {
ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasLogUrl()) {
return null;
}
return p.getLogUrl();
}
@Override
public void setLogURL(String logURL) {
maybeInitBuilder();
if (logURL == null) {
builder.clearLogUrl();
return;
}
builder.setLogUrl(logURL);
}
@Override
public int getContainerExitStatus() {
ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;
return p.getContainerExitStatus();
}
@Override
public ContainerState getContainerState() {
ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasContainerState()) {
return null;
}
return convertFromProtoFormat(p.getContainerState());
}
@Override
public void setContainerState(ContainerState state) {
maybeInitBuilder();
if (state == null) {
builder.clearContainerState();
return;
}
builder.setContainerState(convertToProtoFormat(state));
}
@Override
public void setContainerExitStatus(int containerExitStatus) {
maybeInitBuilder();
builder.setContainerExitStatus(containerExitStatus);
}
public ContainerFinishDataProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
@Override
public int hashCode() {
return getProto().hashCode();
}
@Override
public boolean equals(Object other) {
if (other == null)
return false;
if (other.getClass().isAssignableFrom(this.getClass())) {
return this.getProto().equals(this.getClass().cast(other).getProto());
}
return false;
}
@Override
public String toString() {
return TextFormat.shortDebugString(getProto());
}
private void mergeLocalToBuilder() {
if (this.containerId != null && !((ContainerIdPBImpl)
this.containerId).getProto().equals(builder.getContainerId())) {
builder.setContainerId(convertToProtoFormat(this.containerId));
}
}
private void mergeLocalToProto() {
if (viaProto) {
maybeInitBuilder();
}
mergeLocalToBuilder();
proto = builder.build();
viaProto = true;
}
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ContainerFinishDataProto.newBuilder(proto);
}
viaProto = false;
}
private ContainerIdProto convertToProtoFormat(
ContainerId containerId) {
return ((ContainerIdPBImpl) containerId).getProto();
}
private ContainerIdPBImpl convertFromProtoFormat(
ContainerIdProto containerId) {
return new ContainerIdPBImpl(containerId);
}
private ContainerStateProto convertToProtoFormat(
ContainerState state) {
return ProtoUtils.convertToProtoFormat(state);
}
private ContainerState convertFromProtoFormat(
ContainerStateProto containerState) {
return ProtoUtils.convertFromProtoFormat(containerState);
}
}

137
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerHistoryDataPBImpl.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerStartDataPBImpl.java
View File

@ -19,33 +19,30 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.PriorityPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerHistoryDataProto;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerHistoryDataProtoOrBuilder;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerStartDataProto;
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerStartDataProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerStartData;
import com.google.protobuf.TextFormat;
public class ContainerHistoryDataPBImpl
extends ProtoBase<ContainerHistoryDataProto>
implements ContainerHistoryData {
public class ContainerStartDataPBImpl
extends ContainerStartData {
ContainerHistoryDataProto proto =
ContainerHistoryDataProto.getDefaultInstance();
ContainerHistoryDataProto.Builder builder = null;
ContainerStartDataProto proto =
ContainerStartDataProto.getDefaultInstance();
ContainerStartDataProto.Builder builder = null;
boolean viaProto = false;
private ContainerId containerId;
@ -53,21 +50,21 @@ public class ContainerHistoryDataPBImpl
private NodeId nodeId;
private Priority priority;
public ContainerHistoryDataPBImpl() {
builder = ContainerHistoryDataProto.newBuilder();
public ContainerStartDataPBImpl() {
builder = ContainerStartDataProto.newBuilder();
}
public ContainerHistoryDataPBImpl(ContainerHistoryDataProto proto) {
public ContainerStartDataPBImpl(ContainerStartDataProto proto) {
this.proto = proto;
viaProto = true;
}
@Override
public ContainerId getContainerId() {
if (this != null) {
if (this.containerId != null) {
return this.containerId;
}
ContainerHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasContainerId()) {
return null;
}
@ -86,10 +83,10 @@ public class ContainerHistoryDataPBImpl
@Override
public Resource getAllocatedResource() {
if (this != null) {
if (this.resource != null) {
return this.resource;
}
ContainerHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasAllocatedResource()) {
return null;
}
@ -108,10 +105,10 @@ public class ContainerHistoryDataPBImpl
@Override
public NodeId getAssignedNode() {
if (this != null) {
if (this.nodeId != null) {
return this.nodeId;
}
ContainerHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasAssignedNodeId()) {
return null;
}
@ -130,10 +127,10 @@ public class ContainerHistoryDataPBImpl
@Override
public Priority getPriority() {
if (this != null) {
if (this.priority != null) {
return this.priority;
}
ContainerHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasPriority()) {
return null;
}
@ -152,7 +149,7 @@ public class ContainerHistoryDataPBImpl
@Override
public long getStartTime() {
ContainerHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder;
return p.getStartTime();
}
@ -162,79 +159,7 @@ public class ContainerHistoryDataPBImpl
builder.setStartTime(startTime);
}
@Override
public long getFinishTime() {
ContainerHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
return p.getFinishTime();
}
@Override
public void setFinishTime(long finishTime) {
maybeInitBuilder();
builder.setFinishTime(finishTime);
}
@Override
public String getDiagnosticsInfo() {
ContainerHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasDiagnosticsInfo()) {
return null;
}
return p.getDiagnosticsInfo();
}
@Override
public void setDiagnosticsInfo(String diagnosticInfo) {
maybeInitBuilder();
if (diagnosticInfo == null) {
builder.clearDiagnosticsInfo();
return;
}
builder.setDiagnosticsInfo(diagnosticInfo);
}
@Override
public String getLogURL() {
ContainerHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasLogUrl()) {
return null;
}
return p.getLogUrl();
}
@Override
public void setLogURL(String logURL) {
maybeInitBuilder();
if (logURL == null) {
builder.clearLogUrl();
return;
}
builder.setLogUrl(logURL);
}
@Override
public ContainerState getFinalContainerStatus() {
ContainerHistoryDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasFinalContainerStatus()) {
return null;
}
return convertFromProtoFormat(p.getFinalContainerStatus());
}
@Override
public void setFinalContainerStatus(
ContainerState finalContainerState) {
maybeInitBuilder();
if (finalContainerState == null) {
builder.clearFinalContainerStatus();
return;
}
builder.setFinalContainerStatus(
convertToProtoFormat(finalContainerState));
}
@Override
public ContainerHistoryDataProto getProto() {
public ContainerStartDataProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
@ -258,7 +183,7 @@ public class ContainerHistoryDataPBImpl
@Override
public String toString() {
return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " ");
return TextFormat.shortDebugString(getProto());
}
private void mergeLocalToBuilder() {
@ -291,13 +216,13 @@ public class ContainerHistoryDataPBImpl
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ContainerHistoryDataProto.newBuilder(proto);
builder = ContainerStartDataProto.newBuilder(proto);
}
viaProto = false;
}
private ContainerIdProto convertToProtoFormat(
ContainerId ContainerId) {
ContainerId containerId) {
return ((ContainerIdPBImpl) containerId).getProto();
}
@ -330,14 +255,4 @@ public class ContainerHistoryDataPBImpl
return new PriorityPBImpl(priority);
}
private ContainerState convertFromProtoFormat(
ContainerStateProto finalContainerStatus) {
return ProtoUtils.convertFromProtoFormat(finalContainerStatus);
}
private ContainerStateProto convertToProtoFormat(
ContainerState finalContainerStatus) {
return ProtoUtils.convertToProtoFormat(finalContainerStatus);
}
}

92
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java Normal file
View File

@ -0,0 +1,92 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.applicationhistoryservice;
import java.io.IOException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptStartData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationFinishData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationStartData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerFinishData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerStartData;
public class ApplicationHistoryStoreTestUtils {
protected ApplicationHistoryStore store;
protected void writeApplicationStartData(ApplicationId appId)
throws IOException {
store.applicationStarted(
ApplicationStartData.newInstance(appId, appId.toString(), "test type",
"test queue",
"test user", 0, 0));
}
protected void writeApplicationFinishData(ApplicationId appId)
throws IOException {
store.applicationFinished(
ApplicationFinishData.newInstance(
appId, 0, appId.toString(), FinalApplicationStatus.UNDEFINED,
YarnApplicationState.FINISHED));
}
protected void writeApplicationAttemptStartData(
ApplicationAttemptId appAttemptId) throws IOException {
store.applicationAttemptStarted(
ApplicationAttemptStartData.newInstance(
appAttemptId, appAttemptId.toString(), 0,
ContainerId.newInstance(appAttemptId, 1)));
}
protected void writeApplicationAttemptFinishData(
ApplicationAttemptId appAttemptId) throws IOException {
store.applicationAttemptFinished(
ApplicationAttemptFinishData.newInstance(appAttemptId,
appAttemptId.toString(), "test diagnostics info",
FinalApplicationStatus.UNDEFINED,
YarnApplicationAttemptState.FINISHED));
}
protected void writeContainerStartData(ContainerId containerId)
throws IOException {
store.containerStarted(
ContainerStartData.newInstance(containerId, Resource.newInstance(0, 0),
NodeId.newInstance("localhost", 0),
Priority.newInstance(containerId.getId()), 0));
}
protected void writeContainerFinishData(ContainerId containerId)
throws IOException {
store.containerFinished(
ContainerFinishData.newInstance(containerId, 0, containerId.toString(),
"http://localhost:0/", 0, ContainerState.COMPLETE));
}
}

243
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
View File

@ -19,102 +19,189 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice;
import java.io.IOException;
import java.util.HashMap;
import junit.framework.Assert;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ApplicationAttemptHistoryDataPBImpl;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ApplicationHistoryDataPBImpl;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ContainerHistoryDataPBImpl;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
public class TestMemoryApplicationHistoryStore {
MemoryApplicationHistoryStore memstore = null;
public class TestMemoryApplicationHistoryStore extends
ApplicationHistoryStoreTestUtils {
@Before
public void setup() throws Throwable {
memstore = MemoryApplicationHistoryStore.getMemoryStore();
writeHistoryApplication();
writeHistoryApplicationAttempt();
writeContainer();
}
public void writeHistoryApplication() throws Throwable {
ApplicationHistoryData appData = new ApplicationHistoryDataPBImpl();
appData.setApplicationId(ApplicationId.newInstance(1234, 1));
memstore.writeApplication(appData);
}
public void writeHistoryApplicationAttempt() throws Throwable {
ApplicationAttemptHistoryData appAttemptHistoryData =
new ApplicationAttemptHistoryDataPBImpl();
appAttemptHistoryData.setApplicationAttemptId(ApplicationAttemptId
.newInstance(ApplicationId.newInstance(1234, 1), 1));
memstore.writeApplicationAttempt(appAttemptHistoryData);
}
public void writeContainer() throws Throwable {
ContainerHistoryData container = new ContainerHistoryDataPBImpl();
container.setContainerId(ContainerId.newInstance(ApplicationAttemptId
.newInstance(ApplicationId.newInstance(1234, 1), 1), 1));
memstore.writeContainer(container);
}
public ContainerHistoryData writeContainer(ApplicationAttemptId appAttemptId,
int containerId) throws Throwable {
ContainerHistoryData container = new ContainerHistoryDataPBImpl();
container
.setContainerId(ContainerId.newInstance(appAttemptId, containerId));
memstore.writeContainer(container);
return container;
}
@After
public void tearDown() {
public void setup() {
store = new MemoryApplicationHistoryStore();
}
@Test
public void testReadApplication() throws Throwable {
HashMap<ApplicationId, ApplicationHistoryData> map =
(HashMap<ApplicationId, ApplicationHistoryData>) memstore
.getAllApplications();
Assert.assertEquals(1, map.size());
ApplicationHistoryData appData = null;
for (ApplicationId appId : map.keySet()) {
appData = map.get(appId);
Assert.assertEquals("application_1234_0001", appData.getApplicationId()
.toString());
public void testReadWriteApplicationHistory() throws Exception {
// Out of order
ApplicationId appId = ApplicationId.newInstance(0, 1);
try {
writeApplicationFinishData(appId);
Assert.fail();
} catch (IOException e) {
Assert.assertTrue(e.getMessage().contains(
"is stored before the start information"));
}
HashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttempts =
(HashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>) memstore
.getApplicationAttempts(appData.getApplicationId());
Assert.assertEquals(1, appAttempts.size());
ApplicationAttemptHistoryData appAttempt = null;
for (ApplicationAttemptId appAttemptId : appAttempts.keySet()) {
appAttempt = appAttempts.get(appAttemptId);
Assert.assertEquals("appattempt_1234_0001_000001", appAttempt
.getApplicationAttemptId().toString());
// Normal
int numApps = 5;
for (int i = 1; i <= numApps; ++i) {
appId = ApplicationId.newInstance(0, i);
writeApplicationStartData(appId);
writeApplicationFinishData(appId);
}
Assert.assertEquals(numApps, store.getAllApplications().size());
for (int i = 1; i <= numApps; ++i) {
appId = ApplicationId.newInstance(0, i);
ApplicationHistoryData data = store.getApplication(appId);
Assert.assertNotNull(data);
Assert.assertEquals(appId.toString(), data.getApplicationName());
Assert.assertEquals(appId.toString(), data.getDiagnosticsInfo());
}
// Write again
appId = ApplicationId.newInstance(0, 1);
try {
writeApplicationStartData(appId);
Assert.fail();
} catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("is already stored"));
}
try {
writeApplicationFinishData(appId);
Assert.fail();
} catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("is already stored"));
}
ContainerHistoryData amContainer =
memstore.getContainer(ContainerId.newInstance(ApplicationAttemptId
.newInstance(ApplicationId.newInstance(1234, 1), 1), 1));
Assert.assertEquals("container_1234_0001_01_000001", amContainer
.getContainerId().toString());
ContainerHistoryData container2 =
writeContainer(appAttempt.getApplicationAttemptId(), 2);
HashMap<ContainerId, ContainerHistoryData> containers =
(HashMap<ContainerId, ContainerHistoryData>) memstore
.getContainers(appAttempt.getApplicationAttemptId());
Assert.assertEquals(2, containers.size());
Assert.assertEquals("container_1234_0001_01_000002", containers.get(
container2.getContainerId()).getContainerId().toString());
}
@Test
public void testReadWriteApplicationAttemptHistory() throws Exception {
// Out of order
ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
try {
writeApplicationAttemptFinishData(appAttemptId);
Assert.fail();
} catch (IOException e) {
Assert.assertTrue(e.getMessage().contains(
"is stored before the start information"));
}
// Normal
int numAppAttempts = 5;
writeApplicationStartData(appId);
for (int i = 1; i <= numAppAttempts; ++i) {
appAttemptId =
ApplicationAttemptId.newInstance(appId, i);
writeApplicationAttemptStartData(appAttemptId);
writeApplicationAttemptFinishData(appAttemptId);
}
Assert.assertEquals(
numAppAttempts, store.getApplicationAttempts(appId).size());
for (int i = 1; i <= numAppAttempts; ++i) {
appAttemptId =
ApplicationAttemptId.newInstance(appId, i);
ApplicationAttemptHistoryData data =
store.getApplicationAttempt(appAttemptId);
Assert.assertNotNull(data);
Assert.assertEquals(appAttemptId.toString(), data.getHost());
Assert.assertEquals(appAttemptId.toString(), data.getDiagnosticsInfo());
}
writeApplicationFinishData(appId);
// Write again
appAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
try {
writeApplicationAttemptStartData(appAttemptId);
Assert.fail();
} catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("is already stored"));
}
try {
writeApplicationAttemptFinishData(appAttemptId);
Assert.fail();
} catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("is already stored"));
}
}
@Test
public void testReadWriteContainerHistory() throws Exception {
// Out of order
ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
try {
writeContainerFinishData(containerId);
Assert.fail();
} catch (IOException e) {
Assert.assertTrue(e.getMessage().contains(
"is stored before the start information"));
}
// Normal
writeApplicationAttemptStartData(appAttemptId);
int numContainers = 5;
for (int i = 1; i <= numContainers; ++i) {
containerId = ContainerId.newInstance(appAttemptId, i);
writeContainerStartData(containerId);
writeContainerFinishData(containerId);
}
Assert.assertEquals(
numContainers, store.getContainers(appAttemptId).size());
for (int i = 1; i <= numContainers; ++i) {
containerId = ContainerId.newInstance(appAttemptId, i);
ContainerHistoryData data = store.getContainer(containerId);
Assert.assertNotNull(data);
Assert.assertEquals(Priority.newInstance(containerId.getId()),
data.getPriority());
Assert.assertEquals(containerId.toString(), data.getDiagnosticsInfo());
}
ContainerHistoryData masterContainer = store.getAMContainer(appAttemptId);
Assert.assertNotNull(masterContainer);
Assert.assertEquals(ContainerId.newInstance(appAttemptId, 1),
masterContainer.getContainerId());
writeApplicationAttemptFinishData(appAttemptId);
// Write again
containerId = ContainerId.newInstance(appAttemptId, 1);
try {
writeContainerStartData(containerId);
Assert.fail();
} catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("is already stored"));
}
try {
writeContainerFinishData(containerId);
Assert.fail();
} catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("is already stored"));
}
}
@Test
public void testMassiveWriteContainerHistory() throws IOException {
long mb = 1024 * 1024;
Runtime runtime = Runtime.getRuntime();
long usedMemoryBefore = (runtime.totalMemory() - runtime.freeMemory()) / mb;
int numContainers = 100000;
ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
for (int i = 1; i <= numContainers; ++i) {
ContainerId containerId = ContainerId.newInstance(appAttemptId, i);
writeContainerStartData(containerId);
writeContainerFinishData(containerId);
}
long usedMemoryAfter = (runtime.totalMemory() - runtime.freeMemory()) / mb;
Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 100);
}
}

92
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
View File

@ -33,9 +33,14 @@ import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
@ -43,6 +48,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
* Utility methods to aid serving RM data through the REST and RPC APIs
*/
public class RMServerUtils {
public static List<RMNode> queryRMNodes(RMContext context,
EnumSet<NodeState> acceptedStates) {
// nodes contains nodes that are NEW, RUNNING OR UNHEALTHY
@ -56,7 +62,7 @@ public class RMServerUtils {
}
}
}
// inactiveNodes contains nodes that are DECOMMISSIONED, LOST, OR REBOOTED
if (acceptedStates.contains(NodeState.DECOMMISSIONED) ||
acceptedStates.contains(NodeState.LOST) ||
@ -69,7 +75,7 @@ public class RMServerUtils {
}
return results;
}
/**
* Utility method to validate a list resource requests, by insuring that the
* requested memory/vcore is non-negative and not greater than max
@ -85,8 +91,9 @@ public class RMServerUtils {
* @throw <code>InvalidResourceBlacklistRequestException </code> if the
* resource is not able to be added to the blacklist.
*/
public static void validateBlacklistRequest(ResourceBlacklistRequest blacklistRequest)
throws InvalidResourceBlacklistRequestException {
public static void validateBlacklistRequest(
ResourceBlacklistRequest blacklistRequest)
throws InvalidResourceBlacklistRequestException {
if (blacklistRequest != null) {
List<String> plus = blacklistRequest.getBlacklistAdditions();
if (plus != null && plus.contains(ResourceRequest.ANY)) {
@ -100,10 +107,12 @@ public class RMServerUtils {
* It will validate to make sure all the containers belong to correct
* application attempt id. If not then it will throw
* {@link InvalidContainerReleaseException}
* @param containerReleaseList containers to be released as requested by
* application master.
* @param appAttemptId Application attempt Id
* @throws InvalidContainerReleaseException
*
* @param containerReleaseList
* containers to be released as requested by application master.
* @param appAttemptId
* Application attempt Id
* @throws InvalidContainerReleaseException
*/
public static void
validateContainerReleaseRequest(List<ContainerId> containerReleaseList,
@ -111,9 +120,11 @@ public class RMServerUtils {
throws InvalidContainerReleaseException {
for (ContainerId cId : containerReleaseList) {
if (!appAttemptId.equals(cId.getApplicationAttemptId())) {
throw new InvalidContainerReleaseException("Cannot release container : "
+ cId.toString() + " not belonging to this application attempt : "
+ appAttemptId);
throw new InvalidContainerReleaseException(
"Cannot release container : "
+ cId.toString()
+ " not belonging to this application attempt : "
+ appAttemptId);
}
}
}
@ -157,4 +168,63 @@ public class RMServerUtils {
}
return user;
}
public static YarnApplicationState createApplicationState(
RMAppState rmAppState) {
switch (rmAppState) {
case NEW:
return YarnApplicationState.NEW;
case NEW_SAVING:
return YarnApplicationState.NEW_SAVING;
case SUBMITTED:
return YarnApplicationState.SUBMITTED;
case ACCEPTED:
return YarnApplicationState.ACCEPTED;
case RUNNING:
return YarnApplicationState.RUNNING;
case FINISHING:
case FINISHED:
return YarnApplicationState.FINISHED;
case KILLED:
return YarnApplicationState.KILLED;
case FAILED:
return YarnApplicationState.FAILED;
default:
throw new YarnRuntimeException("Unknown state passed!");
}
}
public static YarnApplicationAttemptState createApplicationAttemptState(
RMAppAttemptState rmAppAttemptState) {
switch (rmAppAttemptState) {
case NEW:
return YarnApplicationAttemptState.NEW;
case SUBMITTED:
return YarnApplicationAttemptState.SUBMITTED;
case SCHEDULED:
return YarnApplicationAttemptState.SCHEDULED;
case ALLOCATED:
return YarnApplicationAttemptState.ALLOCATED;
case LAUNCHED:
return YarnApplicationAttemptState.LAUNCHED;
case ALLOCATED_SAVING:
case LAUNCHED_UNMANAGED_SAVING:
return YarnApplicationAttemptState.ALLOCATED_SAVING;
case RUNNING:
return YarnApplicationAttemptState.RUNNING;
case FINISHING:
return YarnApplicationAttemptState.FINISHING;
case FINAL_SAVING:
return YarnApplicationAttemptState.FINAL_SAVING;
case FINISHED:
return YarnApplicationAttemptState.FINISHED;
case KILLED:
return YarnApplicationAttemptState.KILLED;
case FAILED:
return YarnApplicationAttemptState.FAILED;
default:
throw new YarnRuntimeException("Unknown state passed!");
}
}
}

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
View File

@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
@ -1069,27 +1070,7 @@ public class RMAppImpl implements RMApp, Recoverable {
if (rmAppState.equals(RMAppState.KILLING)) {
rmAppState = stateBeforeKilling;
}
switch (rmAppState) {
case NEW:
return YarnApplicationState.NEW;
case NEW_SAVING:
return YarnApplicationState.NEW_SAVING;
case SUBMITTED:
return YarnApplicationState.SUBMITTED;
case ACCEPTED:
return YarnApplicationState.ACCEPTED;
case RUNNING:
return YarnApplicationState.RUNNING;
case FINISHING:
case FINISHED:
return YarnApplicationState.FINISHED;
case KILLED:
return YarnApplicationState.KILLED;
case FAILED:
return YarnApplicationState.FAILED;
default:
throw new YarnRuntimeException("Unknown state passed!");
}
return RMServerUtils.createApplicationState(rmAppState);
}
public static boolean isAppInFinalState(RMApp rmApp) {