Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-1023. Added Webservices REST APIs support for Application History. Contributed Zhijie Shen. 

svn merge --ignore-ancestry -c 1556749 ../YARN-321


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1562203 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2014-01-28 20:01:12 +00:00
parent b7a3d0ae93
commit c31729d1d4
13 changed files with 974 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -511,6 +511,9 @@ Branch YARN-321: Generic ApplicationHistoryService
YARN-967. Added the client and CLI interfaces for obtaining ApplicationHistory
data. (Mayank Bansal via vinodkv)
YARN-1023. Added Webservices REST APIs support for Application History. (Zhijie
Shen via vinodkv)
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES

147
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
View File

@ -18,22 +18,159 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
import javax.ws.rs.Path;
import java.util.Collections;
import java.util.Set;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.server.api.ApplicationContext;
import org.apache.hadoop.yarn.server.webapp.WebServices;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@Singleton
@Path("/ws/v1/applicationhistory")
public class AHSWebServices {
private ApplicationContext appContext;
public class AHSWebServices extends WebServices {
@Inject
public AHSWebServices(ApplicationContext appContext) {
this.appContext = appContext;
super(appContext);
}
@GET
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public AppsInfo get(
@Context HttpServletRequest req,
@Context HttpServletResponse res) {
return getApps(req, res, null, Collections.<String> emptySet(), null, null,
null, null, null, null, null, null, Collections.<String> emptySet());
}
@GET
@Path("/apps")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
@Override
public AppsInfo getApps(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@QueryParam("state") String stateQuery,
@QueryParam("states") Set<String> statesQuery,
@QueryParam("finalStatus") String finalStatusQuery,
@QueryParam("user") String userQuery,
@QueryParam("queue") String queueQuery,
@QueryParam("limit") String count,
@QueryParam("startedTimeBegin") String startedBegin,
@QueryParam("startedTimeEnd") String startedEnd,
@QueryParam("finishedTimeBegin") String finishBegin,
@QueryParam("finishedTimeEnd") String finishEnd,
@QueryParam("applicationTypes") Set<String> applicationTypes) {
init(res);
validateStates(stateQuery, statesQuery);
return super.getApps(req, res, stateQuery, statesQuery, finalStatusQuery,
userQuery, queueQuery, count, startedBegin, startedEnd, finishBegin,
finishEnd, applicationTypes);
}
@GET
@Path("/apps/{appid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
@Override
public AppInfo getApp(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("appid") String appId) {
init(res);
return super.getApp(req, res, appId);
}
@GET
@Path("/apps/{appid}/appattempts")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
@Override
public AppAttemptsInfo getAppAttempts(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("appid") String appId) {
init(res);
return super.getAppAttempts(req, res, appId);
}
@GET
@Path("/apps/{appid}/appattempts/{appattemptid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
@Override
public AppAttemptInfo getAppAttempt(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("appid") String appId,
@PathParam("appattemptid") String appAttemptId) {
init(res);
return super.getAppAttempt(req, res, appId, appAttemptId);
}
@GET
@Path("/apps/{appid}/appattempts/{appattemptid}/containers")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
@Override
public ContainersInfo getContainers(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("appid") String appId,
@PathParam("appattemptid") String appAttemptId) {
init(res);
return super.getContainers(req, res, appId, appAttemptId);
}
@GET
@Path("/apps/{appid}/appattempts/{appattemptid}/containers/{containerid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
@Override
public ContainerInfo getContainer(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("appid") String appId,
@PathParam("appattemptid") String appAttemptId,
@PathParam("containerid") String containerId) {
init(res);
return super.getContainer(req, res, appId, appAttemptId, containerId);
}
private static void validateStates(
String stateQuery, Set<String> statesQuery) {
// stateQuery is deprecated.
if (stateQuery != null && !stateQuery.isEmpty()) {
statesQuery.add(stateQuery);
}
Set<String> appStates = parseQueries(statesQuery, true);
for (String appState : appStates) {
switch (YarnApplicationState.valueOf(appState.toUpperCase())) {
case FINISHED:
case FAILED:
case KILLED:
continue;
default:
throw new BadRequestException(
"Invalid application-state " + appState
+ " specified. It should be a final state");
}
}
}
}

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/JAXBContextResolver.java
View File

@ -27,8 +27,11 @@ import javax.ws.rs.ext.Provider;
import javax.xml.bind.JAXBContext;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
import com.google.inject.Singleton;
import com.sun.jersey.api.json.JSONConfiguration;
@ -43,7 +46,9 @@ public class JAXBContextResolver implements ContextResolver<JAXBContext> {
private final Set<Class> types;
// you have to specify all the dao classes here
private final Class[] cTypes = { AppInfo.class, AppAttemptInfo.class, ContainerInfo.class };
private final Class[] cTypes = { AppInfo.class, AppsInfo.class,
AppAttemptInfo.class, AppAttemptsInfo.class, ContainerInfo.class,
ContainersInfo.class };
public JAXBContextResolver() throws Exception {
this.types = new HashSet<Class>(Arrays.asList(cTypes));

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java
View File

@ -69,7 +69,7 @@ public class ApplicationHistoryStoreTestUtils {
ApplicationAttemptId appAttemptId) throws IOException {
store.applicationAttemptFinished(
ApplicationAttemptFinishData.newInstance(appAttemptId,
appAttemptId.toString(), "test diagnostics info",
appAttemptId.toString(), "test tracking url",
FinalApplicationStatus.UNDEFINED,
YarnApplicationAttemptState.FINISHED));
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebApp.java
View File

@ -44,6 +44,10 @@ import com.google.inject.Injector;
public class TestAHSWebApp extends ApplicationHistoryStoreTestUtils {
public TestAHSWebApp(ApplicationHistoryStore store) {
this.store = store;
}
@Before
public void setup() {
store = new MemoryApplicationHistoryStore();
@ -138,10 +142,10 @@ public class TestAHSWebApp extends ApplicationHistoryStoreTestUtils {
WebAppTests.flushOutput(injector);
}
private ApplicationHistoryManager mockApplicationHistoryManager(
ApplicationHistoryManager mockApplicationHistoryManager(
int numApps, int numAppAttempts, int numContainers) throws Exception {
ApplicationHistoryManager ahManager =
new MockApplicationHistoryManagerImpl();
new MockApplicationHistoryManagerImpl(store);
for (int i = 1; i <= numApps; ++i) {
ApplicationId appId = ApplicationId.newInstance(0, i);
writeApplicationStartData(appId);
@ -161,10 +165,10 @@ public class TestAHSWebApp extends ApplicationHistoryStoreTestUtils {
return ahManager;
}
private class MockApplicationHistoryManagerImpl extends
class MockApplicationHistoryManagerImpl extends
ApplicationHistoryManagerImpl {
public MockApplicationHistoryManagerImpl() {
public MockApplicationHistoryManagerImpl(ApplicationHistoryStore store) {
super();
init(new YarnConfiguration());
start();

295
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java Normal file
View File

@ -0,0 +1,295 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import javax.ws.rs.core.MediaType;
import junit.framework.Assert;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.server.api.ApplicationContext;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.Before;
import org.junit.Test;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.servlet.GuiceServletContextListener;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.JerseyTest;
import com.sun.jersey.test.framework.WebAppDescriptor;
public class TestAHSWebServices extends JerseyTest {
private static ApplicationHistoryManager ahManager;
private Injector injector = Guice.createInjector(new ServletModule() {
@Override
protected void configureServlets() {
bind(JAXBContextResolver.class);
bind(AHSWebServices.class);
bind(GenericExceptionHandler.class);
try{
ahManager = mockApplicationHistoryManager();
} catch (Exception e) {
Assert.fail();
}
bind(ApplicationContext.class).toInstance(ahManager);
serve("/*").with(GuiceContainer.class);
}
});
public class GuiceServletConfig extends GuiceServletContextListener {
@Override
protected Injector getInjector() {
return injector;
}
}
private ApplicationHistoryManager mockApplicationHistoryManager()
throws Exception {
ApplicationHistoryStore store = new MemoryApplicationHistoryStore();
TestAHSWebApp testAHSWebApp = new TestAHSWebApp(store);
ApplicationHistoryManager ahManager =
testAHSWebApp.mockApplicationHistoryManager(5, 5, 5);
return ahManager;
}
public TestAHSWebServices() {
super(new WebAppDescriptor.Builder(
"org.apache.hadoop.yarn.server.applicationhistoryservice.webapp")
.contextListenerClass(GuiceServletConfig.class)
.filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter").servletPath("/").build());
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
}
@Test
public void testInvalidUri() throws JSONException, Exception {
WebResource r = resource();
String responseStr = "";
try {
responseStr = r.path("ws").path("v1").path("applicationhistory")
.path("bogus").accept(MediaType.APPLICATION_JSON).get(String.class);
fail("should have thrown exception on invalid uri");
} catch (UniformInterfaceException ue) {
ClientResponse response = ue.getResponse();
assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
WebServicesTestUtils.checkStringMatch(
"error string exists and shouldn't", "", responseStr);
}
}
@Test
public void testInvalidUri2() throws JSONException, Exception {
WebResource r = resource();
String responseStr = "";
try {
responseStr = r.accept(MediaType.APPLICATION_JSON).get(String.class);
fail("should have thrown exception on invalid uri");
} catch (UniformInterfaceException ue) {
ClientResponse response = ue.getResponse();
assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
WebServicesTestUtils.checkStringMatch(
"error string exists and shouldn't", "", responseStr);
}
}
@Test
public void testInvalidAccept() throws JSONException, Exception {
WebResource r = resource();
String responseStr = "";
try {
responseStr = r.path("ws").path("v1").path("applicationhistory")
.accept(MediaType.TEXT_PLAIN).get(String.class);
fail("should have thrown exception on invalid uri");
} catch (UniformInterfaceException ue) {
ClientResponse response = ue.getResponse();
assertEquals(Status.INTERNAL_SERVER_ERROR,
response.getClientResponseStatus());
WebServicesTestUtils.checkStringMatch(
"error string exists and shouldn't", "", responseStr);
}
}
@Test
public void testAppsQuery() throws Exception {
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1")
.path("applicationhistory").path("apps")
.queryParam("state", YarnApplicationState.FINISHED.toString())
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
JSONArray array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 5, array.length());
}
@Test
public void testSingleApp() throws Exception {
ApplicationId appId = ApplicationId.newInstance(0, 1);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1")
.path("applicationhistory").path("apps").path(appId.toString())
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject app = json.getJSONObject("app");
assertEquals(appId.toString(), app.getString("appId"));
assertEquals(appId.toString(), app.get("name"));
assertEquals(appId.toString(), app.get("diagnosticsInfo"));
assertEquals("test queue", app.get("queue"));
assertEquals("test user", app.get("user"));
assertEquals("test type", app.get("type"));
assertEquals(FinalApplicationStatus.UNDEFINED.toString(),
app.get("finalAppStatus"));
assertEquals(YarnApplicationState.FINISHED.toString(),
app.get("appState"));
}
@Test
public void testMultipleAttempts() throws Exception {
ApplicationId appId = ApplicationId.newInstance(0, 1);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1")
.path("applicationhistory").path("apps").path(appId.toString())
.path("appattempts").accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject appAttempts = json.getJSONObject("appAttempts");
assertEquals("incorrect number of elements", 1, appAttempts.length());
JSONArray array = appAttempts.getJSONArray("appAttempt");
assertEquals("incorrect number of elements", 5, array.length());
}
@Test
public void testSingleAttempt() throws Exception {
ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1")
.path("applicationhistory").path("apps").path(appId.toString())
.path("appattempts").path(appAttemptId.toString())
.accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject appAttempt = json.getJSONObject("appAttempt");
assertEquals(appAttemptId.toString(),
appAttempt.getString("appAttemptId"));
assertEquals(appAttemptId.toString(), appAttempt.getString("host"));
assertEquals(appAttemptId.toString(),
appAttempt.getString("diagnosticsInfo"));
assertEquals("test tracking url", appAttempt.getString("trackingUrl"));
assertEquals(YarnApplicationAttemptState.FINISHED.toString(),
appAttempt.get("appAttemptState"));
}
@Test
public void testMultipleContainers() throws Exception {
ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1")
.path("applicationhistory").path("apps").path(appId.toString())
.path("appattempts").path(appAttemptId.toString())
.path("containers").accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject containers = json.getJSONObject("containers");
assertEquals("incorrect number of elements", 1, containers.length());
JSONArray array = containers.getJSONArray("container");
assertEquals("incorrect number of elements", 5, array.length());
}
@Test
public void testSingleContainer() throws Exception {
ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1")
.path("applicationhistory").path("apps").path(appId.toString())
.path("appattempts").path(appAttemptId.toString())
.path("containers").path(containerId.toString())
.accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject container = json.getJSONObject("container");
assertEquals(containerId.toString(), container.getString("containerId"));
assertEquals(containerId.toString(),
container.getString("diagnosticsInfo"));
assertEquals("0", container.getString("allocatedMB"));
assertEquals("0", container.getString("allocatedVCores"));
assertEquals(NodeId.newInstance("localhost", 0).toString(),
container.getString("assignedNodeId"));
assertEquals(Priority.newInstance(containerId.getId()).toString(),
container.getString("priority"));
assertEquals("http://localhost:0/", container.getString("logUrl"));
assertEquals(ContainerState.COMPLETE.toString(),
container.getString("containerState"));
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
View File

@ -91,8 +91,8 @@ public class ContainerBlock extends HtmlBlock {
container.getFinishedTime()))).
_("Resource:", container.getAllocatedMB() + " Memory, " +
container.getAllocatedVCores() + " VCores").
_("Logs:", container.getLogUrl() == null ?
"#" : root_url(container.getLogUrl()), container.getLogUrl()).
_("Logs:", container.getLogUrl() == null ? "#" : root_url(container.getLogUrl()),
container.getLogUrl() == null ? "N/A" : container.getLogUrl()).
_("Diagnostics:", container.getDiagnosticsInfo());
html._(InfoBlock.class);

378
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java Normal file
View File

@ -0,0 +1,378 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.webapp;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.WebApplicationException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.server.api.ApplicationContext;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
public class WebServices {
protected ApplicationContext appContext;
public WebServices(ApplicationContext appContext) {
this.appContext = appContext;
}
public AppsInfo getApps(
HttpServletRequest req,
HttpServletResponse res,
String stateQuery,
Set<String> statesQuery,
String finalStatusQuery,
String userQuery,
String queueQuery,
String count,
String startedBegin,
String startedEnd,
String finishBegin,
String finishEnd,
Set<String> applicationTypes) {
long num = 0;
boolean checkCount = false;
boolean checkStart = false;
boolean checkEnd = false;
boolean checkAppTypes = false;
boolean checkAppStates = false;
long countNum = 0;
// set values suitable in case both of begin/end not specified
long sBegin = 0;
long sEnd = Long.MAX_VALUE;
long fBegin = 0;
long fEnd = Long.MAX_VALUE;
if (count != null && !count.isEmpty()) {
checkCount = true;
countNum = Long.parseLong(count);
if (countNum <= 0) {
throw new BadRequestException("limit value must be greater then 0");
}
}
if (startedBegin != null && !startedBegin.isEmpty()) {
checkStart = true;
sBegin = Long.parseLong(startedBegin);
if (sBegin < 0) {
throw new BadRequestException("startedTimeBegin must be greater than 0");
}
}
if (startedEnd != null && !startedEnd.isEmpty()) {
checkStart = true;
sEnd = Long.parseLong(startedEnd);
if (sEnd < 0) {
throw new BadRequestException("startedTimeEnd must be greater than 0");
}
}
if (sBegin > sEnd) {
throw new BadRequestException(
"startedTimeEnd must be greater than startTimeBegin");
}
if (finishBegin != null && !finishBegin.isEmpty()) {
checkEnd = true;
fBegin = Long.parseLong(finishBegin);
if (fBegin < 0) {
throw new BadRequestException("finishTimeBegin must be greater than 0");
}
}
if (finishEnd != null && !finishEnd.isEmpty()) {
checkEnd = true;
fEnd = Long.parseLong(finishEnd);
if (fEnd < 0) {
throw new BadRequestException("finishTimeEnd must be greater than 0");
}
}
if (fBegin > fEnd) {
throw new BadRequestException(
"finishTimeEnd must be greater than finishTimeBegin");
}
Set<String> appTypes = parseQueries(applicationTypes, false);
if (!appTypes.isEmpty()) {
checkAppTypes = true;
}
// stateQuery is deprecated.
if (stateQuery != null && !stateQuery.isEmpty()) {
statesQuery.add(stateQuery);
}
Set<String> appStates = parseQueries(statesQuery, true);
if (!appStates.isEmpty()) {
checkAppStates = true;
}
AppsInfo allApps = new AppsInfo();
Collection<ApplicationReport> appReports = null;
try {
appReports = appContext.getAllApplications().values();
} catch (IOException e) {
throw new WebApplicationException(e);
}
for (ApplicationReport appReport : appReports) {
if (checkCount && num == countNum) {
break;
}
if (checkAppStates && !appStates.contains(
appReport.getYarnApplicationState().toString().toLowerCase())) {
continue;
}
if (finalStatusQuery != null && !finalStatusQuery.isEmpty()) {
FinalApplicationStatus.valueOf(finalStatusQuery);
if (!appReport.getFinalApplicationStatus().toString()
.equalsIgnoreCase(finalStatusQuery)) {
continue;
}
}
if (userQuery != null && !userQuery.isEmpty()) {
if (!appReport.getUser().equals(userQuery)) {
continue;
}
}
if (queueQuery != null && !queueQuery.isEmpty()) {
if (!appReport.getQueue().equals(queueQuery)) {
continue;
}
}
if (checkAppTypes && !appTypes.contains(
appReport.getApplicationType().trim().toLowerCase())) {
continue;
}
if (checkStart
&& (appReport.getStartTime() < sBegin ||
appReport.getStartTime() > sEnd)) {
continue;
}
if (checkEnd
&& (appReport.getFinishTime() < fBegin ||
appReport.getFinishTime() > fEnd)) {
continue;
}
AppInfo app = new AppInfo(appReport);
allApps.add(app);
num++;
}
return allApps;
}
public AppInfo getApp(
HttpServletRequest req, HttpServletResponse res, String appId) {
ApplicationId id = parseApplicationId(appId);
ApplicationReport app = null;
try {
app = appContext.getApplication(id);
} catch (IOException e) {
throw new WebApplicationException(e);
}
if (app == null) {
throw new NotFoundException("app with id: " + appId + " not found");
}
return new AppInfo(app);
}
public AppAttemptsInfo getAppAttempts(
HttpServletRequest req, HttpServletResponse res, String appId) {
ApplicationId id = parseApplicationId(appId);
Collection<ApplicationAttemptReport> appAttemptReports = null;
try {
appAttemptReports = appContext.getApplicationAttempts(id).values();
} catch (IOException e) {
throw new WebApplicationException(e);
}
AppAttemptsInfo appAttemptsInfo = new AppAttemptsInfo();
for (ApplicationAttemptReport appAttemptReport : appAttemptReports) {
AppAttemptInfo appAttemptInfo = new AppAttemptInfo(appAttemptReport);
appAttemptsInfo.add(appAttemptInfo);
}
return appAttemptsInfo;
}
public AppAttemptInfo getAppAttempt(
HttpServletRequest req, HttpServletResponse res,
String appId, String appAttemptId) {
ApplicationId aid = parseApplicationId(appId);
ApplicationAttemptId aaid = parseApplicationAttemptId(appAttemptId);
validateIds(aid, aaid, null);
ApplicationAttemptReport appAttempt = null;
try {
appAttempt = appContext.getApplicationAttempt(aaid);
} catch (IOException e) {
throw new WebApplicationException(e);
}
if (appAttempt == null) {
throw new NotFoundException(
"app attempt with id: " + appAttemptId + " not found");
}
return new AppAttemptInfo(appAttempt);
}
public ContainersInfo getContainers(
HttpServletRequest req, HttpServletResponse res,
String appId, String appAttemptId) {
ApplicationId aid = parseApplicationId(appId);
ApplicationAttemptId aaid = parseApplicationAttemptId(appAttemptId);
validateIds(aid, aaid, null);
Collection<ContainerReport> containerReports = null;
try {
containerReports = appContext.getContainers(aaid).values();
} catch (IOException e) {
throw new WebApplicationException(e);
}
ContainersInfo containersInfo = new ContainersInfo();
for (ContainerReport containerReport : containerReports) {
ContainerInfo containerInfo = new ContainerInfo(containerReport);
containersInfo.add(containerInfo);
}
return containersInfo;
}
public ContainerInfo getContainer(
HttpServletRequest req, HttpServletResponse res,
String appId, String appAttemptId, String containerId) {
ApplicationId aid = parseApplicationId(appId);
ApplicationAttemptId aaid = parseApplicationAttemptId(appAttemptId);
ContainerId cid = parseContainerId(containerId);
validateIds(aid, aaid, cid);
ContainerReport container = null;
try {
container = appContext.getContainer(cid);
} catch (IOException e) {
throw new WebApplicationException(e);
}
if (container == null) {
throw new NotFoundException(
"container with id: " + containerId + " not found");
}
return new ContainerInfo(container);
}
protected void init(HttpServletResponse response) {
// clear content type
response.setContentType(null);
}
protected static Set<String> parseQueries(
Set<String> queries, boolean isState) {
Set<String> params = new HashSet<String>();
if (!queries.isEmpty()) {
for (String query : queries) {
if (query != null && !query.trim().isEmpty()) {
String[] paramStrs = query.split(",");
for (String paramStr : paramStrs) {
if (paramStr != null && !paramStr.trim().isEmpty()) {
if (isState) {
try {
// enum string is in the uppercase
YarnApplicationState.valueOf(paramStr.trim().toUpperCase());
} catch (RuntimeException e) {
YarnApplicationState[] stateArray =
YarnApplicationState.values();
String allAppStates = Arrays.toString(stateArray);
throw new BadRequestException(
"Invalid application-state " + paramStr.trim()
+ " specified. It should be one of " + allAppStates);
}
}
params.add(paramStr.trim().toLowerCase());
}
}
}
}
}
return params;
}
protected static ApplicationId parseApplicationId(String appId) {
if (appId == null || appId.isEmpty()) {
throw new NotFoundException("appId, " + appId + ", is empty or null");
}
ApplicationId aid = ConverterUtils.toApplicationId(appId);
if (aid == null) {
throw new NotFoundException("appId is null");
}
return aid;
}
protected static ApplicationAttemptId parseApplicationAttemptId(
String appAttemptId) {
if (appAttemptId == null || appAttemptId.isEmpty()) {
throw new NotFoundException(
"appAttemptId, " + appAttemptId + ", is empty or null");
}
ApplicationAttemptId aaid =
ConverterUtils.toApplicationAttemptId(appAttemptId);
if (aaid == null) {
throw new NotFoundException("appAttemptId is null");
}
return aaid;
}
protected static ContainerId parseContainerId(String containerId) {
if (containerId == null || containerId.isEmpty()) {
throw new NotFoundException(
"containerId, " + containerId + ", is empty or null");
}
ContainerId cid = ConverterUtils.toContainerId(containerId);
if (cid == null) {
throw new NotFoundException("containerId is null");
}
return cid;
}
protected void validateIds(ApplicationId appId,
ApplicationAttemptId appAttemptId,
ContainerId containerId) {
if (!appAttemptId.getApplicationId().equals(appId)) {
throw new NotFoundException("appId and appAttemptId don't match");
}
if (containerId != null
&& !containerId.getApplicationAttemptId().equals(appAttemptId)) {
throw new NotFoundException("appAttemptId and containerId don't match");
}
}
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppAttemptInfo.java
View File

@ -25,7 +25,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
@XmlRootElement(name = "appattempt")
@XmlRootElement(name = "appAttempt")
@XmlAccessorType(XmlAccessType.FIELD)
public class AppAttemptInfo {

47
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppAttemptsInfo.java Normal file
View File

@ -0,0 +1,47 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by joblicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.webapp.dao;
import java.util.ArrayList;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement(name = "appAttempts")
@XmlAccessorType(XmlAccessType.FIELD)
public class AppAttemptsInfo {
@XmlElement(name = "appAttempt")
protected ArrayList<AppAttemptInfo> attempt = new ArrayList<AppAttemptInfo>();
public AppAttemptsInfo() {
// JAXB needs this
}
public void add(AppAttemptInfo info) {
this.attempt.add(info);
}
public ArrayList<AppAttemptInfo> getAttempts() {
return this.attempt;
}
}

44
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppsInfo.java Normal file
View File

@ -0,0 +1,44 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.webapp.dao;
import java.util.ArrayList;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement(name = "apps")
@XmlAccessorType(XmlAccessType.FIELD)
public class AppsInfo {
protected ArrayList<AppInfo> app = new ArrayList<AppInfo>();
public AppsInfo() {
// JAXB needs this
}
public void add(AppInfo appinfo) {
app.add(appinfo);
}
public ArrayList<AppInfo> getApps() {
return app;
}
}

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
View File

@ -24,7 +24,6 @@ import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.util.Times;
@XmlRootElement(name = "container")
@ -35,7 +34,7 @@ public class ContainerInfo {
protected int allocatedMB;
protected int allocatedVCores;
protected String assignedNodeId;
protected Priority priority;
protected int priority;
protected long startedTime;
protected long finishedTime;
protected long elapsedTime;
@ -57,7 +56,7 @@ public class ContainerInfo {
if (container.getAssignedNode() != null) {
assignedNodeId = container.getAssignedNode().toString();
}
priority = container.getPriority();
priority = container.getPriority().getPriority();
startedTime = container.getStartTime();
finishedTime = container.getFinishTime();
elapsedTime = Times.elapsed(startedTime, finishedTime);
@ -83,7 +82,7 @@ public class ContainerInfo {
return assignedNodeId;
}
public Priority getPriority() {
public int getPriority() {
return priority;
}

44
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainersInfo.java Normal file
View File

@ -0,0 +1,44 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.webapp.dao;
import java.util.ArrayList;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement(name = "containers")
@XmlAccessorType(XmlAccessType.FIELD)
public class ContainersInfo {
protected ArrayList<ContainerInfo> container = new ArrayList<ContainerInfo>();
public ContainersInfo() {
// JAXB needs this
}
public void add(ContainerInfo containerInfo) {
container.add(containerInfo);
}
public ArrayList<ContainerInfo> getContainers() {
return container;
}
}