YARN-9677. Make FpgaDevice and GpuDevice classes more similar to each other. Contributed by kevin su

This commit is contained in:
Szilard Nemeth 2019-11-09 14:04:03 +01:00
parent 42fc8884ab
commit 31f172fd96
17 changed files with 160 additions and 146 deletions

View File

@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resourc
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -30,6 +29,8 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaDevice;
import java.io.IOException;
import java.io.Serializable;
@ -145,124 +146,6 @@ public class FpgaResourceAllocator {
}
}
/** A class that represents an FPGA card. */
public static class FpgaDevice implements Serializable {
private static final long serialVersionUID = -4678487141824092751L;
private final String type;
private final int major;
private final int minor;
// the alias device name. Intel use acl number acl0 to acl31
private final String aliasDevName;
// IP file identifier. matrix multiplication for instance (mutable)
private String IPID;
// SHA-256 hash of the uploaded aocx file (mutable)
private String aocxHash;
// cached hash value
private Integer hashCode;
public String getType() {
return type;
}
public int getMajor() {
return major;
}
public int getMinor() {
return minor;
}
public String getIPID() {
return IPID;
}
public String getAocxHash() {
return aocxHash;
}
public void setAocxHash(String hash) {
this.aocxHash = hash;
}
public void setIPID(String IPID) {
this.IPID = IPID;
}
public String getAliasDevName() {
return aliasDevName;
}
public FpgaDevice(String type, int major, int minor, String aliasDevName) {
this.type = Preconditions.checkNotNull(type, "type must not be null");
this.major = major;
this.minor = minor;
this.aliasDevName = Preconditions.checkNotNull(aliasDevName,
"aliasDevName must not be null");
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
FpgaDevice other = (FpgaDevice) obj;
if (aliasDevName == null) {
if (other.aliasDevName != null) {
return false;
}
} else if (!aliasDevName.equals(other.aliasDevName)) {
return false;
}
if (major != other.major) {
return false;
}
if (minor != other.minor) {
return false;
}
if (type == null) {
if (other.type != null) {
return false;
}
} else if (!type.equals(other.type)) {
return false;
}
return true;
}
@Override
public int hashCode() {
if (hashCode == null) {
final int prime = 31;
int result = 1;
result = prime * result + major;
result = prime * result + type.hashCode();
result = prime * result + minor;
result = prime * result + aliasDevName.hashCode();
hashCode = result;
}
return hashCode;
}
@Override
public String toString() {
return "FPGA Device:(Type: " + this.type + ", Major: " +
this.major + ", Minor: " + this.minor + ", IPID: " +
this.IPID + ", Hash: " + this.aocxHash + ")";
}
}
// called once during initialization
public synchronized void addFpgaDevices(String type, List<FpgaDevice> list) {
availableFpgas.putIfAbsent(type, new LinkedList<>());

View File

@ -42,7 +42,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.AbstractFpgaVendorPlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaDiscoverer;
import org.slf4j.Logger;

View File

@ -19,16 +19,13 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
import java.util.List;
import java.util.Map;
/**
* FPGA plugin interface for vendor to implement. Used by {@link FpgaDiscoverer} and
@ -60,7 +57,7 @@ public interface AbstractFpgaVendorPlugin {
* @param timeout The vendor plugin should return result during this time
* @return The result will be added to FPGAResourceAllocator for later scheduling
* */
List<FpgaResourceAllocator.FpgaDevice> discover(int timeout);
List<FpgaDevice> discover(int timeout);
/**
* Since all vendor plugins share a {@link org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator}

View File

@ -23,7 +23,6 @@ import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin.InnerShellExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

View File

@ -0,0 +1,142 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga;
import java.io.Serializable;
import com.google.common.base.Preconditions;
/** A class that represents an FPGA card. */
public class FpgaDevice implements Serializable {
private static final long serialVersionUID = -4678487141824092751L;
private final String type;
private final int major;
private final int minor;
// the alias device name. Intel use acl number acl0 to acl31
private final String aliasDevName;
// IP file identifier. matrix multiplication for instance (mutable)
private String IPID;
// SHA-256 hash of the uploaded aocx file (mutable)
private String aocxHash;
// cached hash value
private Integer hashCode;
public String getType() {
return type;
}
public int getMajor() {
return major;
}
public int getMinor() {
return minor;
}
public String getIPID() {
return IPID;
}
public String getAocxHash() {
return aocxHash;
}
public void setAocxHash(String hash) {
this.aocxHash = hash;
}
public void setIPID(String IPID) {
this.IPID = IPID;
}
public String getAliasDevName() {
return aliasDevName;
}
public FpgaDevice(String type, int major, int minor, String aliasDevName) {
this.type = Preconditions.checkNotNull(type, "type must not be null");
this.major = major;
this.minor = minor;
this.aliasDevName = Preconditions.checkNotNull(aliasDevName,
"aliasDevName must not be null");
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
FpgaDevice other = (FpgaDevice) obj;
if (aliasDevName == null) {
if (other.aliasDevName != null) {
return false;
}
} else if (!aliasDevName.equals(other.aliasDevName)) {
return false;
}
if (major != other.major) {
return false;
}
if (minor != other.minor) {
return false;
}
if (type == null) {
if (other.type != null) {
return false;
}
} else if (!type.equals(other.type)) {
return false;
}
return true;
}
@Override
public int hashCode() {
if (hashCode == null) {
final int prime = 31;
int result = 1;
result = prime * result + major;
result = prime * result + type.hashCode();
result = prime * result + minor;
result = prime * result + aliasDevName.hashCode();
hashCode = result;
}
return hashCode;
}
@Override
public String toString() {
return "FPGA Device:(Type: " + this.type + ", Major: " + this.major
+ ", Minor: " + this.minor + ", IPID: " + this.IPID + ", Hash: "
+ this.aocxHash + ")";
}
}

View File

@ -34,8 +34,6 @@ import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.discovery.AoclOutputBasedDiscoveryStrategy;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.discovery.FPGADiscoveryStrategy;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.discovery.ScriptBasedFPGADiscoveryStrategy;
@ -52,7 +50,7 @@ public class FpgaDiscoverer extends Configured {
FpgaDiscoverer.class);
private AbstractFpgaVendorPlugin plugin = null;
private List<FpgaResourceAllocator.FpgaDevice> currentFpgaInfo = null;
private List<FpgaDevice> currentFpgaInfo = null;
private Function<String, Optional<String>> scriptRunner = this::runScript;

View File

@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.slf4j.Logger;

View File

@ -22,8 +22,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -138,8 +136,8 @@ public class IntelFpgaOpenclPlugin implements AbstractFpgaVendorPlugin {
}
@Override
public List<FpgaResourceAllocator.FpgaDevice> discover(int timeout) {
List<FpgaResourceAllocator.FpgaDevice> list = new LinkedList<>();
public List<FpgaDevice> discover(int timeout) {
List<FpgaDevice> list = new LinkedList<>();
String output;
output = getDiagnoseInfo(timeout);
if (null == output) {

View File

@ -22,8 +22,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugi
import java.util.List;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.AbstractFpgaVendorPlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaDiscoverer;
/**

View File

@ -26,7 +26,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaDevice;
/**
* Parses a string which specifies FPGA devices. Multiple devices should be

View File

@ -22,7 +22,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugi
import java.util.List;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaDevice;
/**
* Interface for an FPGA device discovery strategy.

View File

@ -24,7 +24,7 @@ import java.util.Optional;
import java.util.function.Function;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaDevice;
/**
* FPGA device discovery strategy which invokes an external script.

View File

@ -22,7 +22,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugi
import java.util.List;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaDevice;
/**
* FPGA device discovery strategy which parses a string.

View File

@ -60,10 +60,10 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceSet;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaDiscoverer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.util.resource.CustomResourceTypesConfigurationProvider;
import org.junit.After;
@ -73,8 +73,8 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import com.google.common.io.Files;
import com.google.common.io.FileWriteMode;
import com.google.common.io.Files;
public class TestFpgaResourceHandlerImpl {
@Rule

View File

@ -24,7 +24,6 @@ import static org.mockito.Mockito.when;
import java.util.List;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin.InnerShellExecutor;
import org.junit.Test;

View File

@ -38,7 +38,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;

View File

@ -77,7 +77,7 @@ import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.nodemanager.amrmproxy.AMRMProxyTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.numa.NumaResourceAllocation;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.LocalResourceTrackerState;