Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-9289. Backport YARN-7330 for GPU in UI to branch-2

This commit is contained in:
Jonathan Hung 2019-02-08 11:26:59 -08:00
parent 4d82318eb1
commit 7939ae1695
40 changed files with 1210 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
View File

@ -633,4 +633,12 @@
<Method name="getResources" />
<Bug pattern="EI_EXPOSE_REP" />
</Match>
<!-- EQ_OVERRIDING_EQUALS_NOT_SYMMETRIC -->
<Match>
<Class name="org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice" />
<Method name="equals" />
<Bug pattern="EQ_OVERRIDING_EQUALS_NOT_SYMMETRIC" />
</Match>
</FindBugsFilter>

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
View File

@ -18,8 +18,12 @@
package org.apache.hadoop.yarn.api.records;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import com.google.common.collect.Lists;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Public;
@ -212,6 +216,22 @@ public abstract class Resource implements Comparable<Resource> {
return resources;
}
/**
* Get list of resource information, this will be used by JAXB.
* @return list of resources copy.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public List<ResourceInformation> getAllResourcesListCopy() {
List<ResourceInformation> list = new ArrayList<>();
for (ResourceInformation i : resources) {
ResourceInformation ri = new ResourceInformation();
ResourceInformation.copy(i, ri);
list.add(ri);
}
return list;
}
/**
* Get ResourceInformation for a specified resource.
*

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
View File

@ -30,11 +30,13 @@ import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@ -224,7 +226,20 @@ public class GpuResourceAllocator {
}
@VisibleForTesting
public synchronized Map<GpuDevice, ContainerId> getDeviceAllocationMapping() {
return new HashMap<>(usedDevices);
public synchronized Map<GpuDevice, ContainerId> getDeviceAllocationMappingCopy() {
return new HashMap<>(usedDevices);
}
public synchronized List<GpuDevice> getAllowedGpusCopy() {
return new ArrayList<>(allowedGpuDevices);
}
public synchronized List<AssignedGpuDevice> getAssignedGpusCopy() {
List<AssignedGpuDevice> assigns = new ArrayList<>();
for (Map.Entry<GpuDevice, ContainerId> entry : usedDevices.entrySet()) {
assigns.add(new AssignedGpuDevice(entry.getKey().getIndex(),
entry.getKey().getMinorNumber(), entry.getValue()));
}
return assigns;
}
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
View File

@ -133,7 +133,6 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
return ret;
}
@VisibleForTesting
public GpuResourceAllocator getGpuAllocator() {
return gpuAllocator;
}

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java
View File

@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain;
import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo;
/**
* {@link ResourcePlugin} is an interface for node manager to easier support
@ -80,4 +81,14 @@ public interface ResourcePlugin {
* @throws YarnException if any issue occurs
*/
void cleanup() throws YarnException;
/**
* Get resource information from this plugin.
*
* @return NMResourceInfo, an example is
* {@link org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation}
*
* @throws YarnException when any issue occurs
*/
NMResourceInfo getNMResourceInfo() throws YarnException;
}

88
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/AssignedGpuDevice.java Normal file
View File

@ -0,0 +1,88 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.yarn.api.records.ContainerId;
/**
* In addition to {@link GpuDevice}, this include container id and more runtime
* information related to who is using the GPU device if possible
*/
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
public class AssignedGpuDevice extends GpuDevice {
private static final long serialVersionUID = -12983712986315L;
String containerId;
public AssignedGpuDevice() {
}
public AssignedGpuDevice(int index, int minorNumber,
ContainerId containerId) {
super(index, minorNumber);
this.containerId = containerId.toString();
}
public String getContainerId() {
return containerId;
}
public void setContainerId(String containerId) {
this.containerId = containerId;
}
@Override
public boolean equals(Object obj) {
if (obj == null || !(obj instanceof AssignedGpuDevice)) {
return false;
}
AssignedGpuDevice other = (AssignedGpuDevice) obj;
return index == other.index && minorNumber == other.minorNumber
&& containerId.equals(other.containerId);
}
@Override
public int compareTo(Object obj) {
if (obj == null || (!(obj instanceof AssignedGpuDevice))) {
return -1;
}
AssignedGpuDevice other = (AssignedGpuDevice) obj;
int result = Integer.compare(index, other.index);
if (0 != result) {
return result;
}
result = Integer.compare(minorNumber, other.minorNumber);
if (0 != result) {
return result;
}
return containerId.compareTo(other.containerId);
}
@Override
public int hashCode() {
final int prime = 47;
return prime * (prime * index + minorNumber) + containerId.hashCode();
}
}

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java
View File

@ -19,15 +19,25 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu;
import java.io.Serializable;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
/**
* This class is used to represent GPU device while allocation.
*/
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
public class GpuDevice implements Serializable, Comparable {
private int index;
private int minorNumber;
protected int index;
protected int minorNumber;
private static final long serialVersionUID = -6812314470754667710L;
public GpuDevice() {
}
public GpuDevice(int index, int minorNumber) {
this.index = index;
this.minorNumber = minorNumber;

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java
View File

@ -18,17 +18,25 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu.GpuResourceAllocator;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu.GpuResourceHandlerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo;
import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation;
import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.NMGpuResourceInfo;
import java.util.List;
import java.util.Map;
public class GpuResourcePlugin implements ResourcePlugin {
private ResourceHandler gpuResourceHandler = null;
private GpuResourceHandlerImpl gpuResourceHandler = null;
private GpuNodeResourceUpdateHandler resourceDiscoverHandler = null;
@Override
@ -58,4 +66,18 @@ public class GpuResourcePlugin implements ResourcePlugin {
public void cleanup() throws YarnException {
// Do nothing.
}
@Override
public NMResourceInfo getNMResourceInfo() throws YarnException {
GpuDeviceInformation gpuDeviceInformation =
GpuDiscoverer.getInstance().getGpuDeviceInformation();
GpuResourceAllocator gpuResourceAllocator =
gpuResourceHandler.getGpuAllocator();
List<GpuDevice> totalGpus = gpuResourceAllocator.getAllowedGpusCopy();
List<AssignedGpuDevice> assignedGpuDevices =
gpuResourceAllocator.getAssignedGpusCopy();
return new NMGpuResourceInfo(gpuDeviceInformation, totalGpus,
assignedGpuDevices);
}
}

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
View File

@ -27,6 +27,10 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -488,6 +492,29 @@ public class NMWebServices {
}
}
@GET
@Path("/resources/{resourcename}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public Object getNMResourceInfo(
@PathParam("resourcename")
String resourceName) throws YarnException {
init();
ResourcePluginManager rpm = this.nmContext.getResourcePluginManager();
if (rpm != null && rpm.getNameToPlugins() != null) {
ResourcePlugin plugin = rpm.getNameToPlugins().get(resourceName);
if (plugin != null) {
NMResourceInfo nmResourceInfo = plugin.getNMResourceInfo();
if (nmResourceInfo != null) {
return nmResourceInfo;
}
}
}
throw new YarnException(
"Could not get detailed resource information for given resource-name="
+ resourceName);
}
private long parseLongParam(String bytes) {
if (bytes == null || bytes.isEmpty()) {
return Long.MAX_VALUE;

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMResourceInfo.java Normal file
View File

@ -0,0 +1,28 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.nodemanager.webapp.dao;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
public class NMResourceInfo {
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java
View File

@ -25,7 +25,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import java.util.List;
/**
* All GPU Device Information in the system.
* All GPU Device Information in the system, fetched from nvidia-smi.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable

80
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/NMGpuResourceInfo.java Normal file
View File

@ -0,0 +1,80 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice;
import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo;
import java.util.List;
/**
* Gpu device information return to client when
* {@link org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices#getNMResourceInfo(String)}
* is invoked.
*/
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
public class NMGpuResourceInfo extends NMResourceInfo {
GpuDeviceInformation gpuDeviceInformation;
List<GpuDevice> totalGpuDevices;
List<AssignedGpuDevice> assignedGpuDevices;
public NMGpuResourceInfo() {
}
public NMGpuResourceInfo(GpuDeviceInformation gpuDeviceInformation,
List<GpuDevice> totalGpuDevices,
List<AssignedGpuDevice> assignedGpuDevices) {
this.gpuDeviceInformation = gpuDeviceInformation;
this.totalGpuDevices = totalGpuDevices;
this.assignedGpuDevices = assignedGpuDevices;
}
public GpuDeviceInformation getGpuDeviceInformation() {
return gpuDeviceInformation;
}
public void setGpuDeviceInformation(
GpuDeviceInformation gpuDeviceInformation) {
this.gpuDeviceInformation = gpuDeviceInformation;
}
public List<GpuDevice> getTotalGpuDevices() {
return totalGpuDevices;
}
public void setTotalGpuDevices(List<GpuDevice> totalGpuDevices) {
this.totalGpuDevices = totalGpuDevices;
}
public List<AssignedGpuDevice> getAssignedGpuDevices() {
return assignedGpuDevices;
}
public void setAssignedGpuDevices(
List<AssignedGpuDevice> assignedGpuDevices) {
this.assignedGpuDevices = assignedGpuDevices;
}
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java
View File

@ -135,7 +135,7 @@ public class PerGpuDeviceInformation {
this.gpuUtilizations = utilizations;
}
@XmlElement(name = "bar1_memory_usage")
@XmlElement(name = "fb_memory_usage")
public PerGpuMemoryUsage getGpuMemoryUsage() {
return gpuMemoryUsage;
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java
View File

@ -27,7 +27,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
@InterfaceAudience.Private
@InterfaceStability.Unstable
@XmlRootElement(name = "bar1_memory_usage")
@XmlRootElement(name = "fb_memory_usage")
public class PerGpuMemoryUsage {
long usedMemoryMiB = -1L;
long availMemoryMiB = -1L;

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
View File

@ -374,7 +374,7 @@ public class TestGpuResourceHandler {
gpuResourceHandler.reacquireContainer(getContainerId(1));
Map<GpuDevice, ContainerId> deviceAllocationMapping =
gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping();
gpuResourceHandler.getGpuAllocator().getDeviceAllocationMappingCopy();
Assert.assertEquals(2, deviceAllocationMapping.size());
Assert.assertTrue(
deviceAllocationMapping.keySet().contains(new GpuDevice(1, 1)));
@ -408,7 +408,7 @@ public class TestGpuResourceHandler {
// Make sure internal state not changed.
deviceAllocationMapping =
gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping();
gpuResourceHandler.getGpuAllocator().getDeviceAllocationMappingCopy();
Assert.assertEquals(2, deviceAllocationMapping.size());
Assert.assertTrue(deviceAllocationMapping.keySet()
.containsAll(Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 3))));
@ -440,7 +440,7 @@ public class TestGpuResourceHandler {
// Make sure internal state not changed.
deviceAllocationMapping =
gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping();
gpuResourceHandler.getGpuAllocator().getDeviceAllocationMappingCopy();
Assert.assertEquals(2, deviceAllocationMapping.size());
Assert.assertTrue(deviceAllocationMapping.keySet()
.containsAll(Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 3))));

188
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
View File

@ -18,25 +18,20 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringReader;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.util.List;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.MediaType;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.junit.Assert;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.servlet.GuiceServletContextListener;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
@ -48,6 +43,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType;
import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo;
import org.apache.hadoop.yarn.logaggregation.TestContainerLogsUtils;
@ -59,7 +55,15 @@ import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice;
import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp;
import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo;
import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation;
import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.NMGpuResourceInfo;
import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.PerGpuDeviceInformation;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams;
@ -73,6 +77,7 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.w3c.dom.Document;
@ -80,24 +85,35 @@ import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.servlet.GuiceServletContextListener;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.MediaType;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringReader;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
/**
* Test the nodemanager node info web services api's
*/
public class TestNMWebServices extends JerseyTestBase {
private static Context nmContext;
private static NodeManager.NMContext nmContext;
private static ResourceView resourceView;
private static ApplicationACLsManager aclsManager;
private static LocalDirsHandlerService dirsHandler;
@ -411,6 +427,116 @@ public class TestNMWebServices extends JerseyTestBase {
assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID));
}
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
private static class MockNMResourceInfo extends NMResourceInfo {
public long a = 1000L;
public MockNMResourceInfo() { }
}
@Test
public void testGetNMResourceInfo()
throws YarnException, InterruptedException, JSONException {
ResourcePluginManager rpm = mock(ResourcePluginManager.class);
Map<String, ResourcePlugin> namesToPlugins = new HashMap<>();
ResourcePlugin mockPlugin1 = mock(ResourcePlugin.class);
NMResourceInfo nmResourceInfo1 = new MockNMResourceInfo();
when(mockPlugin1.getNMResourceInfo()).thenReturn(nmResourceInfo1);
namesToPlugins.put("resource-1", mockPlugin1);
namesToPlugins.put("yarn.io/resource-1", mockPlugin1);
ResourcePlugin mockPlugin2 = mock(ResourcePlugin.class);
namesToPlugins.put("resource-2", mockPlugin2);
when(rpm.getNameToPlugins()).thenReturn(namesToPlugins);
nmContext.setResourcePluginManager(rpm);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("node").path(
"resources").path("resource-2").accept(MediaType.APPLICATION_JSON).get(
ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON, response.getType().toString());
// Access resource-2 should fail (null NMResourceInfo returned).
JSONObject json = response.getEntity(JSONObject.class);
assertIncludesException(json);
// Access resource-3 should fail (unkown plugin)
response = r.path("ws").path("v1").path("node").path(
"resources").path("resource-3").accept(MediaType.APPLICATION_JSON).get(
ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON, response.getType().toString());
json = response.getEntity(JSONObject.class);
assertIncludesException(json);
// Access resource-1 should success
response = r.path("ws").path("v1").path("node").path(
"resources").path("resource-1").accept(MediaType.APPLICATION_JSON).get(
ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON, response.getType().toString());
json = response.getEntity(JSONObject.class);
assertEquals(1000, Long.parseLong(json.get("a").toString()));
// Access resource-1 should success (encoded yarn.io/Fresource-1).
response = r.path("ws").path("v1").path("node").path("resources").path(
"yarn.io%2Fresource-1").accept(MediaType.APPLICATION_JSON).get(
ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON, response.getType().toString());
json = response.getEntity(JSONObject.class);
assertEquals(1000, Long.parseLong(json.get("a").toString()));
}
private ContainerId createContainerId(int id) {
ApplicationId appId = ApplicationId.newInstance(0, 0);
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
ContainerId containerId = ContainerId.newContainerId(appAttemptId, id);
return containerId;
}
@Test
public void testGetYarnGpuResourceInfo()
throws YarnException, InterruptedException, JSONException {
ResourcePluginManager rpm = mock(ResourcePluginManager.class);
Map<String, ResourcePlugin> namesToPlugins = new HashMap<>();
ResourcePlugin mockPlugin1 = mock(ResourcePlugin.class);
GpuDeviceInformation gpuDeviceInformation = new GpuDeviceInformation();
gpuDeviceInformation.setDriverVersion("1.2.3");
gpuDeviceInformation.setGpus(Arrays.asList(new PerGpuDeviceInformation()));
NMResourceInfo nmResourceInfo1 = new NMGpuResourceInfo(gpuDeviceInformation,
Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 2),
new GpuDevice(3, 3)), Arrays
.asList(new AssignedGpuDevice(2, 2, createContainerId(1)),
new AssignedGpuDevice(3, 3, createContainerId(2))));
when(mockPlugin1.getNMResourceInfo()).thenReturn(nmResourceInfo1);
namesToPlugins.put("resource-1", mockPlugin1);
namesToPlugins.put("yarn.io/resource-1", mockPlugin1);
ResourcePlugin mockPlugin2 = mock(ResourcePlugin.class);
namesToPlugins.put("resource-2", mockPlugin2);
when(rpm.getNameToPlugins()).thenReturn(namesToPlugins);
nmContext.setResourcePluginManager(rpm);
WebResource r = resource();
ClientResponse response;
JSONObject json;
// Access resource-1 should success
response = r.path("ws").path("v1").path("node").path(
"resources").path("resource-1").accept(MediaType.APPLICATION_JSON).get(
ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON, response.getType().toString());
json = response.getEntity(JSONObject.class);
assertEquals("1.2.3",
json.getJSONObject("gpuDeviceInformation").get("driver_version"));
assertEquals(3, json.getJSONArray("totalGpuDevices").length());
assertEquals(2, json.getJSONArray("assignedGpuDevices").length());
assertEquals(2, json.getJSONArray("assignedGpuDevices").length());
}
private void assertIncludesException(JSONObject json) {
assertTrue(json.has("RemoteException"));
}
private void testContainerLogs(WebResource r, ContainerId containerId)
throws IOException {
final String containerIdStr = containerId.toString();

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java
View File

@ -39,7 +39,7 @@ public class TestGpuDeviceInformationParser {
Assert.assertEquals(2, info.getGpus().size());
PerGpuDeviceInformation gpu1 = info.getGpus().get(1);
Assert.assertEquals("Tesla P100-PCIE-12GB", gpu1.getProductName());
Assert.assertEquals(16384, gpu1.getGpuMemoryUsage().getTotalMemoryMiB());
Assert.assertEquals(12193, gpu1.getGpuMemoryUsage().getTotalMemoryMiB());
Assert.assertEquals(10.3f,
gpu1.getGpuUtilizations().getOverallGpuUtilization(), 1e-6);
Assert.assertEquals(34f, gpu1.getTemperature().getCurrentGpuTemp(), 1e-6);

33
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-nm-gpu.js Normal file
View File

@ -0,0 +1,33 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import AbstractAdapter from './abstract';
export default AbstractAdapter.extend({
address: "localBaseAddress",
restNameSpace: "node",
serverName: "NM",
urlForFindRecord(id/*, modelName, snapshot*/) {
var url = this._buildURL();
url = url.replace("{nodeAddress}", id) + "/resources/yarn.io%2Fgpu";
return url;
}
});

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js
View File

@ -20,6 +20,7 @@ import Ember from 'ember';
import BaseChartComponent from 'yarn-ui/components/base-chart-component';
import ColorUtils from 'yarn-ui/utils/color-utils';
import Converter from 'yarn-ui/utils/converter';
import {Entities} from 'yarn-ui/constants';
export default BaseChartComponent.extend({
/*
@ -41,8 +42,10 @@ export default BaseChartComponent.extend({
}
if (!middleValue) {
if (this.get("type") === "memory") {
if (this.get(Entities.Type) === Entities.Memory) {
middleValue = Converter.memoryToSimpliedUnit(total);
} else if (this.get(Entities.Type) === Entities.Resource) {
middleValue = Converter.resourceToSimplifiedUnit(total, this.get(Entities.Unit));
} else {
middleValue = total;
}
@ -151,7 +154,10 @@ export default BaseChartComponent.extend({
var value = d.value;
if (this.get("type") === "memory") {
value = Converter.memoryToSimpliedUnit(value);
} else if (this.get("type") === "resource") {
value = Converter.resourceToSimplifiedUnit(value, this.get(Entities.Unit));
}
return d.label + ' = ' + value + suffix;
}.bind(this));
}
@ -185,10 +191,18 @@ export default BaseChartComponent.extend({
}
this.renderDonutChart(this.get("data"), this.get("title"), this.get("showLabels"),
this.get("middleLabel"), this.get("middleValue"));
this.get("middleLabel"), this.get("middleValue"), this.get("suffix"));
},
didInsertElement: function() {
// When parentIdPrefix is specified, use parentidPrefix + name as new parent
// id
if (this.get("parentIdPrefix")) {
var newParentId = this.get("parentIdPrefix") + this.get("id");
this.set("parentId", newParentId);
console.log(newParentId);
}
this.initChart();
this.draw();
},

66
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/gpu-donut-chart.js Normal file
View File

@ -0,0 +1,66 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import DonutChart from 'yarn-ui/components/donut-chart';
import ColorUtils from 'yarn-ui/utils/color-utils';
export default DonutChart.extend({
draw: function() {
// Construct data
var data = [];
if (this.get("gpu-render-type") === "gpu-memory") {
data.push({
label: "Used",
value: parseFloat(this.get("gpuInfo").gpuMemoryUsage.usedMemoryMiB),
});
data.push({
label: "Available",
value: parseFloat(this.get("gpuInfo").gpuMemoryUsage.availMemoryMiB)
});
} else if (this.get("gpu-render-type") === "gpu-utilization") {
var utilization = parseFloat(this.get("gpuInfo").gpuUtilizations.overallGpuUtilization);
data.push({
label: "Utilized",
value: utilization,
});
data.push({
label: "Available",
value: 100 - utilization
});
}
var colorTargets = this.get("colorTargets");
if (colorTargets) {
var colorTargetReverse = Boolean(this.get("colorTargetReverse"));
var targets = colorTargets.split(" ");
this.colors = ColorUtils.getColors(data.length, targets, colorTargetReverse);
}
this.renderDonutChart(data, this.get("title"), this.get("showLabels"),
this.get("middleLabel"), this.get("middleValue"), this.get("suffix"));
},
didInsertElement: function() {
// ParentId includes minorNumber
var newParentId = this.get("parentId") + this.get("gpuInfo").minorNumber;
this.set("parentId", newParentId);
this.initChart();
this.draw();
},
});

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js
View File

@ -22,3 +22,16 @@
export default {
PARAM_SEPARATOR: '!',
};
const BASE_UNIT = 1024
export const Type = 'type';
export const Memory = 'memory';
export const Resource = 'resource';
export const Unit = 'unit';
export const Entities = {
Type: 'type',
Memory:'memory',
Resource: 'resource',
Unit: 'unit'
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
View File

@ -60,7 +60,7 @@ export default Ember.Controller.extend({
getCellContent: function(row) {
var node_id = row.get("id"),
node_addr = row.get("nodeHTTPAddress"),
href = `#/yarn-node/${node_id}/${node_addr}`;
href = `#/yarn-node/${node_id}/${node_addr}/info`;
switch(row.get("nodeState")) {
case "SHUTDOWN":
case "LOST":

69
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
View File

@ -43,6 +43,8 @@ export default DS.Model.extend({
decommissionedNodes: DS.attr('number'),
rebootedNodes: DS.attr('number'),
activeNodes: DS.attr('number'),
totalUsedResourcesAcrossPartition: DS.attr('object'),
totalClusterResourcesAcrossPartition: DS.attr('object'),
getFinishedAppsDataForDonutChart: function() {
var arr = [];
@ -135,4 +137,71 @@ export default DS.Model.extend({
return arr;
}.property("allocatedVirtualCores", "reservedVirtualCores", "availableVirtualCores"),
getResourceTypes: function() {
var types = [];
if (this.get("totalClusterResourcesAcrossPartition")) {
console.log(types);
}
}.property("totalClusterResourcesAcrossPartition"),
/*
* Returned format
* [
* {
* name: <resource-name>
* unit: <resource-unit>
* [
* {
* label: <label>
* value: <value>
* },
* {
* }
* ...
* ],
* }
* ]
*/
getAllResourceTypesDonutChart: function() {
if (this.get("totalClusterResourcesAcrossPartition")
&& this.get("totalUsedResourcesAcrossPartition")) {
var usages = [];
var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourcesInformations;
var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourcesInformations;
clusterResourceInformations.forEach(function(cluster) {
var perResourceTypeUsage = {
name: cluster.name,
unit: cluster.units,
data: []
};
usedResourceInformations.forEach(function (used) {
if (used.name === perResourceTypeUsage.name) {
var usedValue = used.value;
perResourceTypeUsage.data.push({
label: "Used",
value: usedValue
}, {
label: "Available",
value: cluster.value - usedValue
});
}
});
usages.push(perResourceTypeUsage);
// Make sure id is a valid w3c ID
perResourceTypeUsage.id = perResourceTypeUsage.name.replace('/', '-');
perResourceTypeUsage.id = perResourceTypeUsage.id.replace('.', '-');
});
console.log(usages);
return usages;
}
return null;
}.property()
});

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-nm-gpu.js Normal file
View File

@ -0,0 +1,27 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import DS from 'ember-data';
export default DS.Model.extend({
info: DS.attr('object'),
jsonString: function() {
return JSON.stringify(this.get("info"));
}.property(),
});

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
View File

@ -36,6 +36,7 @@ export default DS.Model.extend({
numActiveApplications: DS.attr('number'),
users: DS.hasMany('YarnUser'),
type: DS.attr('string'),
resources: DS.attr('object'),
isLeafQueue: function() {
var len = this.get("children.length");
@ -91,5 +92,5 @@ export default DS.Model.extend({
value: this.get("numActiveApplications") || 0
}
];
}.property("numPendingApplications", "numActiveApplications")
}.property("numPendingApplications", "numActiveApplications"),
});

35
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
View File

@ -32,6 +32,8 @@ export default DS.Model.extend({
availableVirtualCores: DS.attr('number'),
version: DS.attr('string'),
nodeLabels: DS.attr('array'),
availableResource: DS.attr('object'),
usedResource: DS.attr('object'),
nodeLabelsAsString: function() {
var labels = this.get("nodeLabels");
@ -90,6 +92,39 @@ export default DS.Model.extend({
return arr;
}.property("availableVirtualCores", "usedVirtualCores"),
getGpuDataForDonutChart: function() {
var arr = [];
var used = 0;
var ri;
var resourceInformations = this.get("usedResource").resourcesInformations;
for (var i = 0; i < resourceInformations.length; i++) {
ri = resourceInformations[i];
if (ri.name === "yarn.io/gpu") {
used = ri.value;
}
}
var available = 0;
resourceInformations = this.get("availableResource").resourcesInformations;
for (i = 0; i < resourceInformations.length; i++) {
ri = resourceInformations[i];
if (ri.name === "yarn.io/gpu") {
available = ri.value;
}
}
arr.push({
label: "Used",
value: used
});
arr.push({
label: "Available",
value: available
});
return arr;
}.property("availableResource", "usedResource"),
toolTipText: function() {
return "<p>Rack: " + this.get("rack") + '</p>' +
"<p>Host: " + this.get("nodeHostName") + '</p>';

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js
View File

@ -37,7 +37,10 @@ Router.map(function() {
this.route('apps');
});
this.route('yarn-nodes-heatmap');
this.route('yarn-node', { path: '/yarn-node/:node_id/:node_addr' });
this.route('yarn-node', { path: '/yarn-node/:node_id/:node_addr' }, function() {
this.route("info");
this.route("yarn-nm-gpu");
});
this.route('yarn-node-apps', { path: '/yarn-node-apps/:node_id/:node_addr' });
this.route('yarn-node-app',
{ path: '/yarn-node-app/:node_id/:node_addr/:app_id' });

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js
View File

@ -31,7 +31,7 @@ export default AbstractRoute.extend({
queues: this.store.query("yarn-queue.yarn-queue", {}).then((model) => {
let type = model.get('firstObject').get('type');
return this.store.query("yarn-queue." + type + "-queue", {});
}),
})
});
},

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js
View File

@ -25,6 +25,7 @@ export default AbstractRoute.extend({
// Fetches data from both NM and RM. RM is queried to get node usage info.
return Ember.RSVP.hash({
nodeInfo: { id: param.node_id, addr: param.node_addr },
nmGpuInfo: this.store.findRecord('yarn-nm-gpu', param.node_addr, {reload:true}),
node: this.store.findRecord('yarn-node', param.node_addr, {reload: true}),
rmNode: this.store.findRecord('yarn-rm-node', param.node_id, {reload: true})
});
@ -33,5 +34,6 @@ export default AbstractRoute.extend({
unloadAll() {
this.store.unloadAll('yarn-node');
this.store.unloadAll('yarn-rm-node');
this.store.unloadAll('yarn-nm-gpu');
}
});

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node/yarn-nm-gpu.js Normal file
View File

@ -0,0 +1,22 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import Ember from 'ember';
export default Ember.Route.extend({
});

43
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-nm-gpu.js Normal file
View File

@ -0,0 +1,43 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import DS from 'ember-data';
export default DS.JSONAPISerializer.extend({
internalNormalizeSingleResponse(store, primaryModelClass, payload, id) {
if (payload.nodeInfo) {
payload = payload.nodeInfo;
}
var fixedPayload = {
id: id,
type: primaryModelClass.modelName,
attributes: {
info: payload
}
};
return fixedPayload;
},
normalizeSingleResponse(store, primaryModelClass, payload, id/*, requestType*/) {
// payload is of the form {"nodeInfo":{}}
var p = this.internalNormalizeSingleResponse(store,
primaryModelClass, payload, id);
return { data: p };
},
});

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
View File

@ -72,6 +72,7 @@ export default DS.JSONAPISerializer.extend({
preemptionDisabled: payload.preemptionDisabled,
numPendingApplications: payload.numPendingApplications,
numActiveApplications: payload.numActiveApplications,
resources: payload.resources,
type: "capacity",
},
// Relationships

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
View File

@ -41,7 +41,9 @@ export default DS.JSONAPISerializer.extend({
usedVirtualCores: payload.usedVirtualCores,
availableVirtualCores: payload.availableVirtualCores,
version: payload.version,
nodeLabels: payload.nodeLabels
nodeLabels: payload.nodeLabels,
usedResource: payload.used,
availableResource: payload.avail
}
};
return fixedPayload;

88
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/cluster-overview.hbs
View File

@ -90,41 +90,71 @@
<hr>
<div class="row">
<div class="col-lg-4 container-fluid">
<div class="panel panel-default">
<div class="panel-heading">
Resource - Memory
<!-- When getAllResourceTypesDonutChart is not null, use it to show per-resource-type usages. Otherwise only show
vcore/memory usage from metrics -->
{{#if model.clusterMetrics.firstObject.getAllResourceTypesDonutChart}}
{{#each
model.clusterMetrics.firstObject.getAllResourceTypesDonutChart as |perTypeUsage|}}
<div class="col-lg-4 container-fluid">
<div class="panel panel-default">
<div class="panel-heading">
{{perTypeUsage.name}} - Usages
</div>
<div class="container-fluid" id="resource-type-{{perTypeUsage.id}}">
{{donut-chart
data=perTypeUsage.data
showLabels=true
parentIdPrefix="resource-type-"
id=perTypeUsage.id
ratio=0.6
unit=perTypeUsage.unit
type="resource"
maxHeight=350
colorTargets="good"
colorTargetReverse=true}}
</div>
</div>
</div>
<div class="container-fluid" id="mem-donut-chart">
{{donut-chart data=model.clusterMetrics.firstObject.getMemoryDataForDonutChart
showLabels=true
parentId="mem-donut-chart"
ratio=0.6
maxHeight=350
colorTargets="good"
colorTargetReverse=true
type="memory"}}
{{/each}}
{{else}}
<div class="col-lg-4 container-fluid">
<div class="panel panel-default">
<div class="panel-heading">
Resource - Memory
</div>
<div class="container-fluid" id="mem-donut-chart">
{{donut-chart
data=model.clusterMetrics.firstObject.getMemoryDataForDonutChart
showLabels=true
parentId="mem-donut-chart"
ratio=0.6
maxHeight=350
colorTargets="good"
colorTargetReverse=true
type="memory"}}
</div>
</div>
</div>
</div>
<div class="col-lg-4 container-fluid">
<div class="panel panel-default">
<div class="panel-heading">
Resource - VCores
</div>
<div class="container-fluid" id="vcore-donut-chart">
{{donut-chart data=model.clusterMetrics.firstObject.getVCoreDataForDonutChart
showLabels=true
parentId="vcore-donut-chart"
ratio=0.6
maxHeight=350
colorTargets="good"
colorTargetReverse=true}}
<div class="col-lg-4 container-fluid">
<div class="panel panel-default">
<div class="panel-heading">
Resource - VCores
</div>
<div class="container-fluid" id="vcore-donut-chart">
{{donut-chart
data=model.clusterMetrics.firstObject.getVCoreDataForDonutChart
showLabels=true
parentId="vcore-donut-chart"
ratio=0.6
maxHeight=350
colorTargets="good"
colorTargetReverse=true}}
</div>
</div>
</div>
</div>
{{/if}}
</div>
<div class="row">
<div class="col-lg-6 container-fluid">

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs
View File

@ -24,8 +24,8 @@
<div class="panel-body">
<ul class="nav nav-pills nav-stacked" id="stacked-menu">
<ul class="nav nav-pills nav-stacked collapse in">
{{#link-to 'yarn-node' tagName="li"}}
{{#link-to 'yarn-node' nodeId nodeAddr}}Node Information
{{#link-to 'yarn-node.info' tagName="li"}}
{{#link-to 'yarn-node.info' nodeId nodeAddr}}Node Information
{{/link-to}}
{{/link-to}}
{{#link-to 'yarn-node-apps' tagName="li"}}
@ -36,6 +36,12 @@
{{#link-to 'yarn-node-containers' nodeId nodeAddr}}List of Containers
{{/link-to}}
{{/link-to}}
{{#if nmGpuInfo}}
{{#link-to 'yarn-node.yarn-nm-gpu' tagName="li"}}
{{#link-to 'yarn-node.yarn-nm-gpu' nodeId nodeAddr }}GPU Information
{{/link-to}}
{{/link-to}}
{{/if}}
</ul>
</ul>
</div>

69
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-nm-gpu-info.hbs Normal file
View File

@ -0,0 +1,69 @@
{{!
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
}}
<div class="panel panel-default">
<div class="panel-heading">Gpu Information - (Minor
Number {{gpu.minorNumber}})
</div>
<table class="table">
<tbody>
<tr>
<td>Product Name</td>
<td>{{gpu.productName}}</td>
</tr>
<tr>
<td>UUID</td>
<td>{{gpu.uuid}}</td>
</tr>
<tr>
<td>Current Temperature</td>
<td>{{gpu.temperature.currentGpuTemp}}</td>
</tr>
<tr>
<td>Max Temperature</td>
<td>{{gpu.temperature.maxGpuTemp}}</td>
</tr>
</tbody>
</table>
<div class="col-md-5 container-fluid" id="mem-donut-chart{{gpu.minorNumber}}">
{{gpu-donut-chart gpuInfo=gpu
showLabels=true
parentId="mem-donut-chart"
middleLabel = "Gpu Memory"
ratio=0.6
type="memory"
gpu-render-type = "gpu-memory"
colorTargets="good"
colorTargetReverse=true
maxHeight=350}}
</div>
<div class="col-md-5 container-fluid" id="utilization-donut-chart{{gpu.minorNumber}}">
{{gpu-donut-chart gpuInfo=gpu
showLabels=true
parentId="utilization-donut-chart"
middleLabel = "Gpu Utilization"
ratio=0.6
gpu-render-type = "gpu-utilization"
colorTargets="good"
colorTargetReverse=true
suffix="%"
maxHeight=350}}
</div>
</div>

125
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node.hbs
View File

@ -1,125 +0,0 @@
{{!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
--}}
{{breadcrumb-bar breadcrumbs=breadcrumbs}}
<div class="col-md-12 container-fluid">
<div class="row">
{{node-menu-panel path="yarn-node" nodeId=model.rmNode.id nodeAddr=model.node.id}}
<div class="col-md-10 container-fluid">
<div class="row">
<div class="col-md-12 container-fluid">
<div class="panel panel-default">
<div class="panel-heading">Node Information: {{model.rmNode.id}}</div>
<table class="table">
<tbody>
<tr>
<td>Total Vmem allocated for Containers</td>
<td>{{divide num=model.node.totalVmemAllocatedContainersMB den=1024}} GB</td>
</tr>
<tr>
<td>Vmem enforcement enabled</td>
<td>{{model.node.vmemCheckEnabled}}</td>
</tr>
<tr>
<td>Total Pmem allocated for Containers</td>
<td>{{divide num=model.node.totalPmemAllocatedContainersMB den=1024}} GB</td>
</tr>
<tr>
<td>Pmem enforcement enabled</td>
<td>{{model.node.pmemCheckEnabled}}</td>
</tr>
<tr>
<td>Total VCores allocated for Containers</td>
<td>{{model.node.totalVCoresAllocatedContainers}}</td>
</tr>
<tr>
<td>Node Healthy Status</td>
<td>{{model.node.nodeHealthy}}</td>
</tr>
<tr>
<td>Last Node Health Report Time</td>
<td>{{model.node.lastNodeUpdateTime}}</td>
</tr>
<tr>
<td>Node Health Report</td>
<td>{{model.node.healthReport}}</td>
</tr>
{{#if model.node.nmStartupTime}}
<tr>
<td>Node Manager Start Time</td>
<td>{{model.node.nmStartupTime}}</td>
</tr>
{{/if}}
<tr>
<td>Node Manager Version</td>
<td>{{model.node.nodeManagerBuildVersion}}</td>
</tr>
<tr>
<td>Hadoop Version</td>
<td>{{model.node.hadoopBuildVersion}}</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
<div class="row">
<div class="col-lg-6 container-fluid">
<div class="panel panel-default">
<div class="panel-heading">
Resource - Memory
</div>
<div class="container-fluid" id="mem-donut-chart">
{{donut-chart data=model.rmNode.getMemoryDataForDonutChart
showLabels=true
parentId="mem-donut-chart"
ratio=0.6
type="memory"
colorTargets="good"
colorTargetReverse=true
maxHeight=350}}
</div>
</div>
</div>
<div class="col-lg-6 container-fluid">
<div class="panel panel-default">
<div class="panel-heading">
Resource - VCores
</div>
<div class="container-fluid" id="vcore-donut-chart">
{{donut-chart data=model.rmNode.getVCoreDataForDonutChart
showLabels=true
parentId="vcore-donut-chart"
ratio=0.6
colorTargets="good"
colorTargetReverse=true
maxHeight=350}}
</div>
</div>
</div>
</div>
</div>
</div>
</div>
{{outlet}}

154
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/info.hbs Normal file
View File

@ -0,0 +1,154 @@
{{!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
--}}
{{breadcrumb-bar breadcrumbs=breadcrumbs}}
<div class="col-md-12 container-fluid">
<div class="row">
{{node-menu-panel path="yarn-node" nodeId=model.rmNode.id
nodeAddr=model.node.id nmGpuInfo=model.nmGpuInfo}}
<div class="col-md-10 container-fluid">
<div class="row">
<div class="col-md-12 container-fluid">
<div class="panel panel-default">
<div class="panel-heading">Node
Information: {{model.rmNode.id}}</div>
<table class="table">
<tbody>
<tr>
<td>Total Vmem allocated for Containers</td>
<td>{{divide num=model.node.totalVmemAllocatedContainersMB
den=1024}} GB
</td>
</tr>
<tr>
<td>Vmem enforcement enabled</td>
<td>{{model.node.vmemCheckEnabled}}</td>
</tr>
<tr>
<td>Total Pmem allocated for Containers</td>
<td>{{divide num=model.node.totalPmemAllocatedContainersMB
den=1024}} GB
</td>
</tr>
<tr>
<td>Pmem enforcement enabled</td>
<td>{{model.node.pmemCheckEnabled}}</td>
</tr>
<tr>
<td>Total VCores allocated for Containers</td>
<td>{{model.node.totalVCoresAllocatedContainers}}</td>
</tr>
<tr>
<td>Node Healthy Status</td>
<td>{{model.node.nodeHealthy}}</td>
</tr>
<tr>
<td>Last Node Health Report Time</td>
<td>{{model.node.lastNodeUpdateTime}}</td>
</tr>
<tr>
<td>Node Health Report</td>
<td>{{model.node.healthReport}}</td>
</tr>
{{#if model.node.nmStartupTime}}
<tr>
<td>Node Manager Start Time</td>
<td>{{model.node.nmStartupTime}}</td>
</tr>
{{/if}}
<tr>
<td>Node Manager Version</td>
<td>{{model.node.nodeManagerBuildVersion}}</td>
</tr>
<tr>
<td>Hadoop Version</td>
<td>{{model.node.hadoopBuildVersion}}</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
<div class="row">
<div class="col-lg-6 container-fluid">
<div class="panel panel-default">
<div class="panel-heading">
Resource - Memory
</div>
<div class="container-fluid" id="mem-donut-chart">
{{donut-chart data=model.rmNode.getMemoryDataForDonutChart
showLabels=true
parentId="mem-donut-chart"
ratio=0.6
type="memory"
colorTargets="good"
colorTargetReverse=true
maxHeight=350}}
</div>
</div>
</div>
<div class="col-lg-6 container-fluid">
<div class="panel panel-default">
<div class="panel-heading">
Resource - VCores
</div>
<div class="container-fluid" id="vcore-donut-chart">
{{donut-chart data=model.rmNode.getVCoreDataForDonutChart
showLabels=true
parentId="vcore-donut-chart"
ratio=0.6
colorTargets="good"
colorTargetReverse=true
maxHeight=350}}
</div>
</div>
</div>
</div>
{{#if model.nmGpuInfo}}
<div class="row">
<div class="col-lg-6 container-fluid">
<div class="panel panel-default">
<div class="panel-heading">
<li>
Resources - yarn.io/gpu
</li>
</div>
<div class="container-fluid" id="gpu-donut-chart">
{{donut-chart data=model.rmNode.getGpuDataForDonutChart
showLabels=true
parentId="gpu-donut-chart"
ratio=0.6
colorTargets="good"
colorTargetReverse=true
maxHeight=350}}
</div>
</div>
</div>
</div>
{{/if}}
</div>
</div>
</div>
{{outlet}}

53
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/yarn-nm-gpu.hbs Normal file
View File

@ -0,0 +1,53 @@
{{!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
--}}
{{breadcrumb-bar breadcrumbs=breadcrumbs}}
<div class="col-md-12 container-fluid">
<div class="row">
{{node-menu-panel path="yarn-node" nodeId=model.rmNode.id
nodeAddr=model.node.id nmGpuInfo=model.nmGpuInfo}}
<div class="col-md-10 container-fluid">
<div class="panel panel-default">
<div class="panel-heading">Gpu Information</div>
<table class="table">
<tbody>
<tr>
<td>Vendor</td>
<td>NVIDIA</td>
</tr>
<tr>
<td>Driver Version</td>
<td>{{model.nmGpuInfo.info.gpuDeviceInformation.driverVersion}}</td>
</tr>
<tr>
<td>Total Number Of Gpus</td>
<td>{{model.nmGpuInfo.info.totalGpuDevices.length}}</td>
</tr>
</tbody>
</table>
</div>
{{#each model.nmGpuInfo.info.gpuDeviceInformation.gpus as |gpu|}}
{{yarn-nm-gpu-info gpu=gpu}}
{{/each}}
</div>
</div>
</div>

51
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js
View File

@ -130,6 +130,57 @@ export default {
}
return value.toFixed(1) + " " + unit;
},
resourceToSimplifiedUnit: function (value, unit) {
// First convert unit to base unit ("").
var normalizedValue = value;
if (unit === "Ki") {
normalizedValue = normalizedValue * 1024;
} else if (unit === "Mi") {
normalizedValue = normalizedValue * 1024 * 1024;
} else if (unit === "Gi") {
normalizedValue = normalizedValue * 1024 * 1024 * 1024;
} else if (unit === "Ti") {
normalizedValue = normalizedValue * 1024 * 1024 * 1024 * 1024;
} else if (unit === "Pi") {
normalizedValue = normalizedValue * 1024 * 1024 * 1024 * 1024 * 1024;
} else if (unit === "K" || unit === "k") {
normalizedValue = normalizedValue * 1000;
} else if (unit === "M" || unit === "m") {
normalizedValue = normalizedValue * 1000 * 1000;
} else if (unit === "G" || unit === "g") {
normalizedValue = normalizedValue * 1000 * 1000 * 1000;
} else if (unit === "T" || unit === "t") {
normalizedValue = normalizedValue * 1000 * 1000 * 1000 * 1000;
} else if (unit === "P" || unit === "p") {
normalizedValue = normalizedValue * 1000 * 1000 * 1000 * 1000 * 1000;
}
// From baseunit ("") convert to most human readable unit
// (which value < 1024 * 0.9).
var finalUnit = "";
if (normalizedValue / 1024 >= 0.9) {
normalizedValue = normalizedValue / 1024;
finalUnit = "Ki";
}
if (normalizedValue / 1024 >= 0.9) {
normalizedValue = normalizedValue / 1024;
finalUnit = "Mi";
}
if (normalizedValue / 1024 >= 0.9) {
normalizedValue = normalizedValue / 1024;
finalUnit = "Gi";
}
if (normalizedValue / 1024 >= 0.9) {
normalizedValue = normalizedValue / 1024;
finalUnit = "Ti";
}
if (normalizedValue / 1024 >= 0.9) {
normalizedValue = normalizedValue / 1024;
finalUnit = "Pi";
}
return normalizedValue.toFixed(1) + " " + finalUnit;
},
msToElapsedTimeUnit: function(millisecs, short) {
var seconds = Math.floor(millisecs / 1000);
var days = Math.floor(seconds / (3600 * 24));