YARN-6679. Reduce Resource instance overhead via non-PBImpl
(Contributed by Daryn Sharp via Daniel Templeton)
This commit is contained in:
parent
2448d84572
commit
c5b28c03a1
@ -23,7 +23,6 @@
|
|||||||
import org.apache.hadoop.classification.InterfaceStability.Evolving;
|
import org.apache.hadoop.classification.InterfaceStability.Evolving;
|
||||||
import org.apache.hadoop.classification.InterfaceStability.Stable;
|
import org.apache.hadoop.classification.InterfaceStability.Stable;
|
||||||
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
|
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -53,22 +52,49 @@
|
|||||||
@Stable
|
@Stable
|
||||||
public abstract class Resource implements Comparable<Resource> {
|
public abstract class Resource implements Comparable<Resource> {
|
||||||
|
|
||||||
|
private static class SimpleResource extends Resource {
|
||||||
|
private long memory;
|
||||||
|
private long vcores;
|
||||||
|
SimpleResource(long memory, long vcores) {
|
||||||
|
this.memory = memory;
|
||||||
|
this.vcores = vcores;
|
||||||
|
}
|
||||||
|
@Override
|
||||||
|
public int getMemory() {
|
||||||
|
return (int)memory;
|
||||||
|
}
|
||||||
|
@Override
|
||||||
|
public void setMemory(int memory) {
|
||||||
|
this.memory = memory;
|
||||||
|
}
|
||||||
|
@Override
|
||||||
|
public long getMemorySize() {
|
||||||
|
return memory;
|
||||||
|
}
|
||||||
|
@Override
|
||||||
|
public void setMemorySize(long memory) {
|
||||||
|
this.memory = memory;
|
||||||
|
}
|
||||||
|
@Override
|
||||||
|
public int getVirtualCores() {
|
||||||
|
return (int)vcores;
|
||||||
|
}
|
||||||
|
@Override
|
||||||
|
public void setVirtualCores(int vcores) {
|
||||||
|
this.vcores = vcores;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Public
|
@Public
|
||||||
@Stable
|
@Stable
|
||||||
public static Resource newInstance(int memory, int vCores) {
|
public static Resource newInstance(int memory, int vCores) {
|
||||||
Resource resource = Records.newRecord(Resource.class);
|
return new SimpleResource(memory, vCores);
|
||||||
resource.setMemorySize(memory);
|
|
||||||
resource.setVirtualCores(vCores);
|
|
||||||
return resource;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Public
|
@Public
|
||||||
@Stable
|
@Stable
|
||||||
public static Resource newInstance(long memory, int vCores) {
|
public static Resource newInstance(long memory, int vCores) {
|
||||||
Resource resource = Records.newRecord(Resource.class);
|
return new SimpleResource(memory, vCores);
|
||||||
resource.setMemorySize(memory);
|
|
||||||
resource.setVirtualCores(vCores);
|
|
||||||
return resource;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -167,6 +193,15 @@ public boolean equals(Object obj) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int compareTo(Resource other) {
|
||||||
|
long diff = this.getMemorySize() - other.getMemorySize();
|
||||||
|
if (diff == 0) {
|
||||||
|
diff = this.getVirtualCores() - other.getVirtualCores();
|
||||||
|
}
|
||||||
|
return diff == 0 ? 0 : (diff > 0 ? 1 : -1);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return "<memory:" + getMemorySize() + ", vCores:" + getVirtualCores() + ">";
|
return "<memory:" + getMemorySize() + ", vCores:" + getVirtualCores() + ">";
|
||||||
|
@ -748,7 +748,7 @@ private synchronized ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private synchronized ResourceProto convertToProtoFormat(Resource r) {
|
private synchronized ResourceProto convertToProtoFormat(Resource r) {
|
||||||
return ((ResourcePBImpl) r).getProto();
|
return ProtoUtils.convertToProtoFormat(r);
|
||||||
}
|
}
|
||||||
|
|
||||||
private synchronized PreemptionMessagePBImpl convertFromProtoFormat(PreemptionMessageProto p) {
|
private synchronized PreemptionMessagePBImpl convertFromProtoFormat(PreemptionMessageProto p) {
|
||||||
|
@ -25,6 +25,7 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
|
||||||
@ -158,13 +159,13 @@ private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
|
|||||||
private ApplicationIdProto convertToProtoFormat(ApplicationId t) {
|
private ApplicationIdProto convertToProtoFormat(ApplicationId t) {
|
||||||
return ((ApplicationIdPBImpl)t).getProto();
|
return ((ApplicationIdPBImpl)t).getProto();
|
||||||
}
|
}
|
||||||
|
|
||||||
private Resource convertFromProtoFormat(ResourceProto resource) {
|
private Resource convertFromProtoFormat(ResourceProto resource) {
|
||||||
return new ResourcePBImpl(resource);
|
return new ResourcePBImpl(resource);
|
||||||
}
|
}
|
||||||
|
|
||||||
private ResourceProto convertToProtoFormat(Resource resource) {
|
private ResourceProto convertToProtoFormat(Resource resource) {
|
||||||
return ((ResourcePBImpl)resource).getProto();
|
return ProtoUtils.convertToProtoFormat(resource);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -438,7 +438,7 @@ private Resource convertFromProtoFormat(ResourceProto resource) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ResourceProto convertToProtoFormat(Resource resource) {
|
private ResourceProto convertToProtoFormat(Resource resource) {
|
||||||
return ((ResourcePBImpl)resource).getProto();
|
return ProtoUtils.convertToProtoFormat(resource);
|
||||||
}
|
}
|
||||||
|
|
||||||
private ContainerPBImpl convertFromProtoFormat(ContainerProto p) {
|
private ContainerPBImpl convertFromProtoFormat(ContainerProto p) {
|
||||||
|
@ -79,20 +79,14 @@ public String toString() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void mergeLocalToBuilder() {
|
private void mergeLocalToBuilder() {
|
||||||
if (this.usedResources != null
|
if (this.usedResources != null) {
|
||||||
&& !((ResourcePBImpl) this.usedResources).getProto().equals(
|
|
||||||
builder.getUsedResources())) {
|
|
||||||
builder.setUsedResources(convertToProtoFormat(this.usedResources));
|
builder.setUsedResources(convertToProtoFormat(this.usedResources));
|
||||||
}
|
}
|
||||||
if (this.reservedResources != null
|
if (this.reservedResources != null) {
|
||||||
&& !((ResourcePBImpl) this.reservedResources).getProto().equals(
|
|
||||||
builder.getReservedResources())) {
|
|
||||||
builder.setReservedResources(
|
builder.setReservedResources(
|
||||||
convertToProtoFormat(this.reservedResources));
|
convertToProtoFormat(this.reservedResources));
|
||||||
}
|
}
|
||||||
if (this.neededResources != null
|
if (this.neededResources != null) {
|
||||||
&& !((ResourcePBImpl) this.neededResources).getProto().equals(
|
|
||||||
builder.getNeededResources())) {
|
|
||||||
builder.setNeededResources(convertToProtoFormat(this.neededResources));
|
builder.setNeededResources(convertToProtoFormat(this.neededResources));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -257,7 +251,7 @@ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ResourceProto convertToProtoFormat(Resource t) {
|
private ResourceProto convertToProtoFormat(Resource t) {
|
||||||
return ((ResourcePBImpl)t).getProto();
|
return ProtoUtils.convertToProtoFormat(t);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -120,9 +120,7 @@ private void mergeLocalToBuilder() {
|
|||||||
if (this.amContainer != null) {
|
if (this.amContainer != null) {
|
||||||
builder.setAmContainerSpec(convertToProtoFormat(this.amContainer));
|
builder.setAmContainerSpec(convertToProtoFormat(this.amContainer));
|
||||||
}
|
}
|
||||||
if (this.resource != null &&
|
if (this.resource != null) {
|
||||||
!((ResourcePBImpl) this.resource).getProto().equals(
|
|
||||||
builder.getResource())) {
|
|
||||||
builder.setResource(convertToProtoFormat(this.resource));
|
builder.setResource(convertToProtoFormat(this.resource));
|
||||||
}
|
}
|
||||||
if (this.applicationTags != null && !this.applicationTags.isEmpty()) {
|
if (this.applicationTags != null && !this.applicationTags.isEmpty()) {
|
||||||
@ -475,7 +473,7 @@ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ResourceProto convertToProtoFormat(Resource t) {
|
private ResourceProto convertToProtoFormat(Resource t) {
|
||||||
return ((ResourcePBImpl)t).getProto();
|
return ProtoUtils.convertToProtoFormat(t);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -93,9 +93,7 @@ private void mergeLocalToBuilder() {
|
|||||||
builder.getNodeId())) {
|
builder.getNodeId())) {
|
||||||
builder.setNodeId(convertToProtoFormat(this.nodeId));
|
builder.setNodeId(convertToProtoFormat(this.nodeId));
|
||||||
}
|
}
|
||||||
if (this.resource != null
|
if (this.resource != null) {
|
||||||
&& !((ResourcePBImpl) this.resource).getProto().equals(
|
|
||||||
builder.getResource())) {
|
|
||||||
builder.setResource(convertToProtoFormat(this.resource));
|
builder.setResource(convertToProtoFormat(this.resource));
|
||||||
}
|
}
|
||||||
if (this.priority != null &&
|
if (this.priority != null &&
|
||||||
@ -307,7 +305,7 @@ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ResourceProto convertToProtoFormat(Resource t) {
|
private ResourceProto convertToProtoFormat(Resource t) {
|
||||||
return ((ResourcePBImpl)t).getProto();
|
return ProtoUtils.convertToProtoFormat(t);
|
||||||
}
|
}
|
||||||
|
|
||||||
private PriorityPBImpl convertFromProtoFormat(PriorityProto p) {
|
private PriorityPBImpl convertFromProtoFormat(PriorityProto p) {
|
||||||
|
@ -270,9 +270,7 @@ private void mergeLocalToBuilder() {
|
|||||||
&& !((NodeIdPBImpl) nodeId).getProto().equals(builder.getNodeId())) {
|
&& !((NodeIdPBImpl) nodeId).getProto().equals(builder.getNodeId())) {
|
||||||
builder.setNodeId(convertToProtoFormat(this.nodeId));
|
builder.setNodeId(convertToProtoFormat(this.nodeId));
|
||||||
}
|
}
|
||||||
if (this.resource != null
|
if (this.resource != null) {
|
||||||
&& !((ResourcePBImpl) this.resource).getProto().equals(
|
|
||||||
builder.getResource())) {
|
|
||||||
builder.setResource(convertToProtoFormat(this.resource));
|
builder.setResource(convertToProtoFormat(this.resource));
|
||||||
}
|
}
|
||||||
if (this.priority != null
|
if (this.priority != null
|
||||||
@ -318,7 +316,7 @@ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ResourceProto convertToProtoFormat(Resource t) {
|
private ResourceProto convertToProtoFormat(Resource t) {
|
||||||
return ((ResourcePBImpl) t).getProto();
|
return ProtoUtils.convertToProtoFormat(t);
|
||||||
}
|
}
|
||||||
|
|
||||||
private PriorityPBImpl convertFromProtoFormat(PriorityProto p) {
|
private PriorityPBImpl convertFromProtoFormat(PriorityProto p) {
|
||||||
|
@ -107,7 +107,7 @@ private Resource convertFromProtoFormat(ResourceProto p) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ResourceProto convertToProtoFormat(Resource t) {
|
private ResourceProto convertToProtoFormat(Resource t) {
|
||||||
return ((ResourcePBImpl) t).getProto();
|
return ProtoUtils.convertToProtoFormat(t);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void mergeLocalToProto() {
|
private void mergeLocalToProto() {
|
||||||
|
@ -131,7 +131,7 @@ private Resource convertFromProtoFormat(ResourceProto p) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ResourceProto convertToProtoFormat(Resource t) {
|
private ResourceProto convertToProtoFormat(Resource t) {
|
||||||
return ((ResourcePBImpl) t).getProto();
|
return ProtoUtils.convertToProtoFormat(t);
|
||||||
}
|
}
|
||||||
|
|
||||||
private Token convertFromProtoFormat(TokenProto p) {
|
private Token convertFromProtoFormat(TokenProto p) {
|
||||||
|
@ -111,7 +111,7 @@ private Resource convertFromProtoFormat(ResourceProto p) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ResourceProto convertToProtoFormat(Resource t) {
|
private ResourceProto convertToProtoFormat(Resource t) {
|
||||||
return ((ResourcePBImpl) t).getProto();
|
return ProtoUtils.convertToProtoFormat(t);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void mergeLocalToProto() {
|
private void mergeLocalToProto() {
|
||||||
|
@ -339,7 +339,7 @@ private ExecutionTypeProto convertToProtoFormat(ExecutionType e) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ResourceProto convertToProtoFormat(Resource e) {
|
private ResourceProto convertToProtoFormat(Resource e) {
|
||||||
return ((ResourcePBImpl)e).getProto();
|
return ProtoUtils.convertToProtoFormat(e);
|
||||||
}
|
}
|
||||||
|
|
||||||
private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
|
private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
|
||||||
|
@ -253,14 +253,10 @@ private void mergeLocalToBuilder() {
|
|||||||
builder.getNodeId())) {
|
builder.getNodeId())) {
|
||||||
builder.setNodeId(convertToProtoFormat(this.nodeId));
|
builder.setNodeId(convertToProtoFormat(this.nodeId));
|
||||||
}
|
}
|
||||||
if (this.used != null
|
if (this.used != null) {
|
||||||
&& !((ResourcePBImpl) this.used).getProto().equals(
|
|
||||||
builder.getUsed())) {
|
|
||||||
builder.setUsed(convertToProtoFormat(this.used));
|
builder.setUsed(convertToProtoFormat(this.used));
|
||||||
}
|
}
|
||||||
if (this.capability != null
|
if (this.capability != null) {
|
||||||
&& !((ResourcePBImpl) this.capability).getProto().equals(
|
|
||||||
builder.getCapability())) {
|
|
||||||
builder.setCapability(convertToProtoFormat(this.capability));
|
builder.setCapability(convertToProtoFormat(this.capability));
|
||||||
}
|
}
|
||||||
if (this.labels != null) {
|
if (this.labels != null) {
|
||||||
@ -310,7 +306,7 @@ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ResourceProto convertToProtoFormat(Resource r) {
|
private ResourceProto convertToProtoFormat(Resource r) {
|
||||||
return ((ResourcePBImpl) r).getProto();
|
return ProtoUtils.convertToProtoFormat(r);
|
||||||
}
|
}
|
||||||
|
|
||||||
private ResourceUtilizationPBImpl convertFromProtoFormat(
|
private ResourceUtilizationPBImpl convertFromProtoFormat(
|
||||||
|
@ -359,7 +359,7 @@ public static ContainerUpdateType convertFromProtoFormat(
|
|||||||
* Resource
|
* Resource
|
||||||
*/
|
*/
|
||||||
public static synchronized ResourceProto convertToProtoFormat(Resource r) {
|
public static synchronized ResourceProto convertToProtoFormat(Resource r) {
|
||||||
return ((ResourcePBImpl) r).getProto();
|
return ResourcePBImpl.getProto(r);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Resource convertFromProtoFormat(ResourceProto resource) {
|
public static Resource convertFromProtoFormat(ResourceProto resource) {
|
||||||
|
@ -140,7 +140,7 @@ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ResourceProto convertToProtoFormat(Resource t) {
|
private ResourceProto convertToProtoFormat(Resource t) {
|
||||||
return ((ResourcePBImpl) t).getProto();
|
return ProtoUtils.convertToProtoFormat(t);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -146,7 +146,7 @@ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ResourceProto convertToProtoFormat(Resource p) {
|
private ResourceProto convertToProtoFormat(Resource p) {
|
||||||
return ((ResourcePBImpl)p).getProto();
|
return ProtoUtils.convertToProtoFormat(p);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void mergeLocalToBuilder() {
|
private void mergeLocalToBuilder() {
|
||||||
|
@ -80,7 +80,7 @@ private void maybeInitBuilder() {
|
|||||||
|
|
||||||
private ResourceProto convertToProtoFormat(
|
private ResourceProto convertToProtoFormat(
|
||||||
Resource resource) {
|
Resource resource) {
|
||||||
return ((ResourcePBImpl)resource).getProto();
|
return ProtoUtils.convertToProtoFormat(resource);
|
||||||
}
|
}
|
||||||
|
|
||||||
private ResourcePBImpl convertFromProtoFormat(
|
private ResourcePBImpl convertFromProtoFormat(
|
||||||
|
@ -31,7 +31,20 @@ public class ResourcePBImpl extends Resource {
|
|||||||
ResourceProto proto = ResourceProto.getDefaultInstance();
|
ResourceProto proto = ResourceProto.getDefaultInstance();
|
||||||
ResourceProto.Builder builder = null;
|
ResourceProto.Builder builder = null;
|
||||||
boolean viaProto = false;
|
boolean viaProto = false;
|
||||||
|
|
||||||
|
// call via ProtoUtils.convertToProtoFormat(Resource)
|
||||||
|
static ResourceProto getProto(Resource r) {
|
||||||
|
final ResourcePBImpl pb;
|
||||||
|
if (r instanceof ResourcePBImpl) {
|
||||||
|
pb = (ResourcePBImpl)r;
|
||||||
|
} else {
|
||||||
|
pb = new ResourcePBImpl();
|
||||||
|
pb.setMemorySize(r.getMemorySize());
|
||||||
|
pb.setVirtualCores(r.getVirtualCores());
|
||||||
|
}
|
||||||
|
return pb.getProto();
|
||||||
|
}
|
||||||
|
|
||||||
public ResourcePBImpl() {
|
public ResourcePBImpl() {
|
||||||
builder = ResourceProto.newBuilder();
|
builder = ResourceProto.newBuilder();
|
||||||
}
|
}
|
||||||
@ -89,15 +102,4 @@ public void setVirtualCores(int vCores) {
|
|||||||
maybeInitBuilder();
|
maybeInitBuilder();
|
||||||
builder.setVirtualCores(vCores);
|
builder.setVirtualCores(vCores);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public int compareTo(Resource other) {
|
|
||||||
long diff = this.getMemorySize() - other.getMemorySize();
|
|
||||||
if (diff == 0) {
|
|
||||||
diff = this.getVirtualCores() - other.getVirtualCores();
|
|
||||||
}
|
|
||||||
return diff == 0 ? 0 : (diff > 0 ? 1 : -1);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -217,7 +217,7 @@ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ResourceProto convertToProtoFormat(Resource t) {
|
private ResourceProto convertToProtoFormat(Resource t) {
|
||||||
return ((ResourcePBImpl)t).getProto();
|
return ProtoUtils.convertToProtoFormat(t);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -132,7 +132,7 @@ public ContainerTokenIdentifier(ContainerId containerID, int containerVersion,
|
|||||||
builder.setNmHostAddr(hostName);
|
builder.setNmHostAddr(hostName);
|
||||||
builder.setAppSubmitter(appSubmitter);
|
builder.setAppSubmitter(appSubmitter);
|
||||||
if (r != null) {
|
if (r != null) {
|
||||||
builder.setResource(((ResourcePBImpl)r).getProto());
|
builder.setResource(ProtoUtils.convertToProtoFormat(r));
|
||||||
}
|
}
|
||||||
builder.setExpiryTimeStamp(expiryTimeStamp);
|
builder.setExpiryTimeStamp(expiryTimeStamp);
|
||||||
builder.setMasterKeyId(masterKeyId);
|
builder.setMasterKeyId(masterKeyId);
|
||||||
|
@ -21,7 +21,6 @@
|
|||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
import org.apache.hadoop.yarn.api.records.*;
|
import org.apache.hadoop.yarn.api.records.*;
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
|
||||||
|
|
||||||
@InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"})
|
@InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"})
|
||||||
@Unstable
|
@Unstable
|
||||||
@ -123,10 +122,7 @@ public static Resource createResource(int memory) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public static Resource createResource(int memory, int cores) {
|
public static Resource createResource(int memory, int cores) {
|
||||||
Resource resource = Records.newRecord(Resource.class);
|
return Resource.newInstance(memory, cores);
|
||||||
resource.setMemorySize(memory);
|
|
||||||
resource.setVirtualCores(cores);
|
|
||||||
return resource;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Resource createResource(long memory) {
|
public static Resource createResource(long memory) {
|
||||||
@ -134,10 +130,7 @@ public static Resource createResource(long memory) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public static Resource createResource(long memory, int cores) {
|
public static Resource createResource(long memory, int cores) {
|
||||||
Resource resource = Records.newRecord(Resource.class);
|
return Resource.newInstance(memory, cores);
|
||||||
resource.setMemorySize(memory);
|
|
||||||
resource.setVirtualCores(cores);
|
|
||||||
return resource;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Resource none() {
|
public static Resource none() {
|
||||||
|
@ -25,6 +25,7 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.PriorityPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.PriorityPBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
|
||||||
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerStartDataProto;
|
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerStartDataProto;
|
||||||
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerStartDataProtoOrBuilder;
|
import org.apache.hadoop.yarn.proto.ApplicationHistoryServerProtos.ContainerStartDataProtoOrBuilder;
|
||||||
@ -189,9 +190,7 @@ private void mergeLocalToBuilder() {
|
|||||||
builder.getContainerId())) {
|
builder.getContainerId())) {
|
||||||
builder.setContainerId(convertToProtoFormat(this.containerId));
|
builder.setContainerId(convertToProtoFormat(this.containerId));
|
||||||
}
|
}
|
||||||
if (this.resource != null
|
if (this.resource != null) {
|
||||||
&& !((ResourcePBImpl) this.resource).getProto().equals(
|
|
||||||
builder.getAllocatedResource())) {
|
|
||||||
builder.setAllocatedResource(convertToProtoFormat(this.resource));
|
builder.setAllocatedResource(convertToProtoFormat(this.resource));
|
||||||
}
|
}
|
||||||
if (this.nodeId != null
|
if (this.nodeId != null
|
||||||
@ -232,7 +231,7 @@ private ContainerIdProto convertToProtoFormat(ContainerId containerId) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ResourceProto convertToProtoFormat(Resource resource) {
|
private ResourceProto convertToProtoFormat(Resource resource) {
|
||||||
return ((ResourcePBImpl) resource).getProto();
|
return ProtoUtils.convertToProtoFormat(resource);
|
||||||
}
|
}
|
||||||
|
|
||||||
private ResourcePBImpl convertFromProtoFormat(ResourceProto resource) {
|
private ResourcePBImpl convertFromProtoFormat(ResourceProto resource) {
|
||||||
|
@ -256,9 +256,7 @@ private void mergeLocalToBuilder() {
|
|||||||
builder.setContainerId(convertToProtoFormat(this.containerId));
|
builder.setContainerId(convertToProtoFormat(this.containerId));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.resource != null
|
if (this.resource != null) {
|
||||||
&& !((ResourcePBImpl) this.resource).getProto().equals(
|
|
||||||
builder.getResource())) {
|
|
||||||
builder.setResource(convertToProtoFormat(this.resource));
|
builder.setResource(convertToProtoFormat(this.resource));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -295,7 +293,7 @@ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ResourceProto convertToProtoFormat(Resource t) {
|
private ResourceProto convertToProtoFormat(Resource t) {
|
||||||
return ((ResourcePBImpl) t).getProto();
|
return ProtoUtils.convertToProtoFormat(t);
|
||||||
}
|
}
|
||||||
|
|
||||||
private ContainerStateProto
|
private ContainerStateProto
|
||||||
|
@ -598,7 +598,7 @@ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ResourceProto convertToProtoFormat(Resource t) {
|
private ResourceProto convertToProtoFormat(Resource t) {
|
||||||
return ((ResourcePBImpl)t).getProto();
|
return ProtoUtils.convertToProtoFormat(t);
|
||||||
}
|
}
|
||||||
|
|
||||||
private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
|
private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
|
||||||
|
@ -32,6 +32,7 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
|
||||||
@ -387,7 +388,7 @@ private static ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private static ResourceProto convertToProtoFormat(Resource t) {
|
private static ResourceProto convertToProtoFormat(Resource t) {
|
||||||
return ((ResourcePBImpl)t).getProto();
|
return ProtoUtils.convertToProtoFormat(t);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static NMContainerStatusPBImpl convertFromProtoFormat(
|
private static NMContainerStatusPBImpl convertFromProtoFormat(
|
||||||
|
@ -21,6 +21,7 @@
|
|||||||
|
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase;
|
||||||
|
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
|
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
|
||||||
@ -250,7 +251,7 @@ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private ResourceProto convertToProtoFormat(Resource t) {
|
private ResourceProto convertToProtoFormat(Resource t) {
|
||||||
return ((ResourcePBImpl)t).getProto();
|
return ProtoUtils.convertToProtoFormat(t);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -40,6 +40,9 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.Priority;
|
import org.apache.hadoop.yarn.api.records.Priority;
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
|
||||||
|
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
|
||||||
|
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
|
||||||
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NMContainerStatusPBImpl;
|
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NMContainerStatusPBImpl;
|
||||||
|
|
||||||
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb
|
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb
|
||||||
@ -55,6 +58,23 @@
|
|||||||
|
|
||||||
public class TestProtocolRecords {
|
public class TestProtocolRecords {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testResource() {
|
||||||
|
final long mem = 123;
|
||||||
|
final int vcores = 456;
|
||||||
|
final Resource r = Resource.newInstance(mem, vcores);
|
||||||
|
// should be a lightweight SimpleResource which is a private inner class
|
||||||
|
// so just verify it's not the heavyweight pb impl.
|
||||||
|
Assert.assertFalse(r instanceof ResourcePBImpl);
|
||||||
|
Assert.assertEquals(mem, r.getMemorySize());
|
||||||
|
Assert.assertEquals(vcores, r.getVirtualCores());
|
||||||
|
|
||||||
|
ResourceProto proto = ProtoUtils.convertToProtoFormat(r);
|
||||||
|
Assert.assertEquals(mem, proto.getMemory());
|
||||||
|
Assert.assertEquals(vcores, proto.getVirtualCores());
|
||||||
|
Assert.assertEquals(r, ProtoUtils.convertFromProtoFormat(proto));
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testNMContainerStatus() {
|
public void testNMContainerStatus() {
|
||||||
ApplicationId appId = ApplicationId.newInstance(123456789, 1);
|
ApplicationId appId = ApplicationId.newInstance(123456789, 1);
|
||||||
|
@ -44,6 +44,7 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
|
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
|
||||||
@ -365,7 +366,7 @@ public void storeContainerResourceChanged(ContainerId containerId,
|
|||||||
try {
|
try {
|
||||||
// New value will overwrite old values for the same key
|
// New value will overwrite old values for the same key
|
||||||
batch.put(bytes(keyResChng),
|
batch.put(bytes(keyResChng),
|
||||||
((ResourcePBImpl) capability).getProto().toByteArray());
|
ProtoUtils.convertToProtoFormat(capability).toByteArray());
|
||||||
batch.put(bytes(keyVersion), bytes(Integer.toString(containerVersion)));
|
batch.put(bytes(keyVersion), bytes(Integer.toString(containerVersion)));
|
||||||
db.write(batch);
|
db.write(batch);
|
||||||
} finally {
|
} finally {
|
||||||
|
@ -33,6 +33,7 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.LogAggregationContextPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.LogAggregationContextPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.PriorityPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.PriorityPBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
|
||||||
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
|
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.LogAggregationContextProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.LogAggregationContextProto;
|
||||||
@ -60,7 +61,7 @@ public ContainerTokenIdentifierForTest(ContainerId containerID,
|
|||||||
builder.setNmHostAddr(hostName);
|
builder.setNmHostAddr(hostName);
|
||||||
builder.setAppSubmitter(appSubmitter);
|
builder.setAppSubmitter(appSubmitter);
|
||||||
if (r != null) {
|
if (r != null) {
|
||||||
builder.setResource(((ResourcePBImpl)r).getProto());
|
builder.setResource(ProtoUtils.convertToProtoFormat(r));
|
||||||
}
|
}
|
||||||
builder.setExpiryTimeStamp(expiryTimeStamp);
|
builder.setExpiryTimeStamp(expiryTimeStamp);
|
||||||
builder.setMasterKeyId(masterKeyId);
|
builder.setMasterKeyId(masterKeyId);
|
||||||
@ -91,7 +92,7 @@ public ContainerTokenIdentifierForTest(ContainerTokenIdentifier identifier,
|
|||||||
|
|
||||||
ResourcePBImpl resource = (ResourcePBImpl)identifier.getResource();
|
ResourcePBImpl resource = (ResourcePBImpl)identifier.getResource();
|
||||||
if (resource != null) {
|
if (resource != null) {
|
||||||
builder.setResource(resource.getProto());
|
builder.setResource(ProtoUtils.convertToProtoFormat(resource));
|
||||||
}
|
}
|
||||||
|
|
||||||
builder.setExpiryTimeStamp(identifier.getExpiryTimeStamp());
|
builder.setExpiryTimeStamp(identifier.getExpiryTimeStamp());
|
||||||
|
Loading…
x
Reference in New Issue
Block a user