YARN-10304. Create an endpoint for remote application log directory path query. Contributed by Andras Gyori
This commit is contained in:
parent
cc641534dc
commit
82a7505646
|
@ -423,6 +423,24 @@ public class HsWebServices extends WebServices {
|
|||
return new JobTaskAttemptCounterInfo(ta);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the user qualified path name of the remote log directory for
|
||||
* each pre-configured log aggregation file controller.
|
||||
*
|
||||
* @param req HttpServletRequest
|
||||
* @return Path names grouped by file controller name
|
||||
*/
|
||||
@GET
|
||||
@Path("/remote-log-dir")
|
||||
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
|
||||
public Response getRemoteLogDirPath(@Context HttpServletRequest req,
|
||||
@QueryParam(YarnWebServiceParams.REMOTE_USER) String user,
|
||||
@QueryParam(YarnWebServiceParams.APP_ID) String appIdStr)
|
||||
throws IOException {
|
||||
init();
|
||||
return logServlet.getRemoteLogDirPath(user, appIdStr);
|
||||
}
|
||||
|
||||
@GET
|
||||
@Path("/aggregatedlogs")
|
||||
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
|
||||
|
|
|
@ -31,6 +31,7 @@ import org.apache.hadoop.fs.Path;
|
|||
import org.apache.hadoop.mapreduce.v2.app.AppContext;
|
||||
import org.apache.hadoop.mapreduce.v2.hs.HistoryContext;
|
||||
import org.apache.hadoop.mapreduce.v2.hs.MockHistoryContext;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
|
||||
|
@ -48,9 +49,13 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
|||
import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType;
|
||||
import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo;
|
||||
import org.apache.hadoop.yarn.logaggregation.TestContainerLogsUtils;
|
||||
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController;
|
||||
import org.apache.hadoop.yarn.logaggregation.filecontroller.ifile.LogAggregationIndexedFileController;
|
||||
import org.apache.hadoop.yarn.server.webapp.LogServlet;
|
||||
import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams;
|
||||
import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo;
|
||||
import org.apache.hadoop.yarn.server.webapp.dao.RemoteLogPathEntry;
|
||||
import org.apache.hadoop.yarn.server.webapp.dao.RemoteLogPaths;
|
||||
import org.apache.hadoop.yarn.webapp.BadRequestException;
|
||||
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
|
||||
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
|
||||
|
@ -68,6 +73,8 @@ import javax.ws.rs.core.MediaType;
|
|||
import java.net.HttpURLConnection;
|
||||
import java.net.URI;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
@ -76,6 +83,8 @@ import java.util.Set;
|
|||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
|
@ -116,6 +125,9 @@ public class TestHsWebServicesLogs extends JerseyTestBase {
|
|||
private static final String USER = "fakeUser";
|
||||
private static final String FILE_NAME = "syslog";
|
||||
|
||||
private static final String REMOTE_LOG_DIR_SUFFIX = "test-logs";
|
||||
private static final String[] FILE_FORMATS = {"IFile", "TFile"};
|
||||
|
||||
private static final String NM_WEBADDRESS_1 = "test-nm-web-address-1:9999";
|
||||
private static final NodeId NM_ID_1 = NodeId.newInstance("fakeHost1", 9951);
|
||||
private static final String NM_WEBADDRESS_2 = "test-nm-web-address-2:9999";
|
||||
|
@ -156,6 +168,17 @@ public class TestHsWebServicesLogs extends JerseyTestBase {
|
|||
}
|
||||
|
||||
private static class WebServletModule extends ServletModule {
|
||||
private Configuration newConf;
|
||||
|
||||
WebServletModule() {
|
||||
super();
|
||||
}
|
||||
|
||||
WebServletModule(Configuration newConf) {
|
||||
super();
|
||||
this.newConf = newConf;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void configureServlets() {
|
||||
MockHistoryContext appContext = new MockHistoryContext(0, 1, 2, 1);
|
||||
|
@ -199,8 +222,9 @@ public class TestHsWebServicesLogs extends JerseyTestBase {
|
|||
fail("Failed to setup WebServletModule class");
|
||||
}
|
||||
|
||||
Configuration usedConf = newConf == null ? conf : newConf;
|
||||
HsWebServices hsWebServices =
|
||||
new HsWebServices(appContext, conf, webApp, mockProtocol);
|
||||
new HsWebServices(appContext, usedConf, webApp, mockProtocol);
|
||||
try {
|
||||
LogServlet logServlet = hsWebServices.getLogServlet();
|
||||
logServlet = spy(logServlet);
|
||||
|
@ -576,6 +600,92 @@ public class TestHsWebServicesLogs extends JerseyTestBase {
|
|||
+ ContainerLogAggregationType.AGGREGATED, "Hello-" + CONTAINER_2_2_3);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRemoteLogDirWithUser() {
|
||||
createReconfiguredServlet();
|
||||
|
||||
WebResource r = resource();
|
||||
ClientResponse response = r.path("ws").path("v1")
|
||||
.path("history").path("remote-log-dir")
|
||||
.queryParam(YarnWebServiceParams.REMOTE_USER,
|
||||
USER)
|
||||
.accept(MediaType.APPLICATION_JSON)
|
||||
.get(ClientResponse.class);
|
||||
RemoteLogPaths res = response.
|
||||
getEntity(new GenericType<RemoteLogPaths>(){});
|
||||
|
||||
List<String> collectedControllerNames = new ArrayList<>();
|
||||
for (RemoteLogPathEntry entry: res.getPaths()) {
|
||||
String path = String.format("%s/%s/bucket-%s-%s",
|
||||
YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR, USER,
|
||||
REMOTE_LOG_DIR_SUFFIX, entry.getFileController().toLowerCase());
|
||||
collectedControllerNames.add(entry.getFileController());
|
||||
assertEquals(entry.getPath(), path);
|
||||
}
|
||||
|
||||
assertTrue(collectedControllerNames.containsAll(
|
||||
Arrays.asList(FILE_FORMATS)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRemoteLogDir() {
|
||||
createReconfiguredServlet();
|
||||
UserGroupInformation ugi = UserGroupInformation.
|
||||
createRemoteUser(USER);
|
||||
UserGroupInformation.setLoginUser(ugi);
|
||||
|
||||
WebResource r = resource();
|
||||
ClientResponse response = r.path("ws").path("v1")
|
||||
.path("history").path("remote-log-dir")
|
||||
.accept(MediaType.APPLICATION_JSON)
|
||||
.get(ClientResponse.class);
|
||||
RemoteLogPaths res = response.
|
||||
getEntity(new GenericType<RemoteLogPaths>(){});
|
||||
|
||||
List<String> collectedControllerNames = new ArrayList<>();
|
||||
for (RemoteLogPathEntry entry: res.getPaths()) {
|
||||
String path = String.format("%s/%s/bucket-%s-%s",
|
||||
YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR, USER,
|
||||
REMOTE_LOG_DIR_SUFFIX, entry.getFileController().toLowerCase());
|
||||
collectedControllerNames.add(entry.getFileController());
|
||||
assertEquals(entry.getPath(), path);
|
||||
}
|
||||
|
||||
assertTrue(collectedControllerNames.containsAll(
|
||||
Arrays.asList(FILE_FORMATS)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRemoteLogDirWithUserAndAppId() {
|
||||
createReconfiguredServlet();
|
||||
|
||||
WebResource r = resource();
|
||||
ClientResponse response = r.path("ws").path("v1")
|
||||
.path("history").path("remote-log-dir")
|
||||
.queryParam(YarnWebServiceParams.REMOTE_USER,
|
||||
USER)
|
||||
.queryParam(YarnWebServiceParams.APP_ID,
|
||||
APPID_1.toString())
|
||||
.accept(MediaType.APPLICATION_JSON)
|
||||
.get(ClientResponse.class);
|
||||
RemoteLogPaths res = response.
|
||||
getEntity(new GenericType<RemoteLogPaths>(){});
|
||||
|
||||
List<String> collectedControllerNames = new ArrayList<>();
|
||||
for (RemoteLogPathEntry entry: res.getPaths()) {
|
||||
String path = String.format("%s/%s/bucket-%s-%s/0001/%s",
|
||||
YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR, USER,
|
||||
REMOTE_LOG_DIR_SUFFIX, entry.getFileController().toLowerCase(),
|
||||
APPID_1.toString());
|
||||
collectedControllerNames.add(entry.getFileController());
|
||||
assertEquals(entry.getPath(), path);
|
||||
}
|
||||
|
||||
assertTrue(collectedControllerNames.containsAll(
|
||||
Arrays.asList(FILE_FORMATS)));
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testNonExistingAppId() {
|
||||
ApplicationId nonExistingApp = ApplicationId.newInstance(99, 99);
|
||||
|
@ -763,4 +873,20 @@ public class TestHsWebServicesLogs extends JerseyTestBase {
|
|||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private void createReconfiguredServlet() {
|
||||
Configuration newConf = new YarnConfiguration();
|
||||
newConf.setStrings(YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS,
|
||||
FILE_FORMATS);
|
||||
newConf.setClass(String.format(
|
||||
YarnConfiguration.LOG_AGGREGATION_FILE_CONTROLLER_FMT, "IFile"),
|
||||
LogAggregationIndexedFileController.class,
|
||||
LogAggregationFileController.class);
|
||||
newConf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
|
||||
YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR);
|
||||
newConf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX,
|
||||
REMOTE_LOG_DIR_SUFFIX);
|
||||
GuiceServletConfig.setInjector(
|
||||
Guice.createInjector(new WebServletModule(newConf)));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -157,6 +157,14 @@ public abstract class LogAggregationFileController {
|
|||
return this.remoteRootLogDirSuffix;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the name of the file controller.
|
||||
* @return name of the file controller.
|
||||
*/
|
||||
public String getFileControllerName() {
|
||||
return this.fileControllerName;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the writer.
|
||||
* @param context the {@link LogAggregationFileControllerContext}
|
||||
|
|
|
@ -24,13 +24,19 @@ import com.sun.jersey.api.client.ClientHandlerException;
|
|||
import com.sun.jersey.api.client.UniformInterfaceException;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.conf.Configured;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
|
||||
import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType;
|
||||
import org.apache.hadoop.yarn.logaggregation.ContainerLogMeta;
|
||||
import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
|
||||
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController;
|
||||
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
|
||||
import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo;
|
||||
import org.apache.hadoop.yarn.server.webapp.dao.RemoteLogPathEntry;
|
||||
import org.apache.hadoop.yarn.server.webapp.dao.RemoteLogPaths;
|
||||
import org.apache.hadoop.yarn.util.Apps;
|
||||
import org.apache.hadoop.yarn.webapp.BadRequestException;
|
||||
import org.apache.hadoop.yarn.webapp.NotFoundException;
|
||||
|
@ -45,6 +51,7 @@ import javax.ws.rs.core.GenericEntity;
|
|||
import javax.ws.rs.core.Response;
|
||||
import javax.ws.rs.core.Response.Status;
|
||||
import javax.ws.rs.core.StreamingOutput;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
|
@ -174,6 +181,47 @@ public class LogServlet extends Configured {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the user qualified path name of the remote log directory for
|
||||
* each pre-configured log aggregation file controller.
|
||||
*
|
||||
* @return {@link Response} object containing remote log dir path names
|
||||
*/
|
||||
public Response getRemoteLogDirPath(String user, String applicationId)
|
||||
throws IOException {
|
||||
String remoteUser = user;
|
||||
ApplicationId appId = applicationId != null ?
|
||||
ApplicationIdPBImpl.fromString(applicationId) : null;
|
||||
|
||||
if (remoteUser == null) {
|
||||
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
|
||||
remoteUser = ugi.getUserName();
|
||||
}
|
||||
|
||||
List<LogAggregationFileController> fileControllers =
|
||||
getOrCreateFactory().getConfiguredLogAggregationFileControllerList();
|
||||
List<RemoteLogPathEntry> paths = new ArrayList<>();
|
||||
|
||||
for (LogAggregationFileController fileController : fileControllers) {
|
||||
String path;
|
||||
if (appId != null) {
|
||||
path = fileController.getRemoteAppLogDir(appId, remoteUser).toString();
|
||||
} else {
|
||||
path = LogAggregationUtils.getRemoteLogSuffixedDir(
|
||||
fileController.getRemoteRootLogDir(),
|
||||
remoteUser, fileController.getRemoteRootLogDirSuffix()).toString();
|
||||
}
|
||||
|
||||
paths.add(new RemoteLogPathEntry(fileController.getFileControllerName(),
|
||||
path));
|
||||
}
|
||||
|
||||
RemoteLogPaths result = new RemoteLogPaths(paths);
|
||||
Response.ResponseBuilder response = Response.ok().entity(result);
|
||||
response.header("X-Content-Type-Options", "nosniff");
|
||||
return response.build();
|
||||
}
|
||||
|
||||
public Response getLogsInfo(HttpServletRequest hsr, String appIdStr,
|
||||
String appAttemptIdStr, String containerIdStr, String nmId,
|
||||
boolean redirectedFromNode, boolean manualRedirection) {
|
||||
|
|
|
@ -39,4 +39,5 @@ public interface YarnWebServiceParams {
|
|||
String REDIRECTED_FROM_NODE = "redirected_from_node";
|
||||
String CLUSTER_ID = "clusterid";
|
||||
String MANUAL_REDIRECTION = "manual_redirection";
|
||||
String REMOTE_USER = "user";
|
||||
}
|
||||
|
|
|
@ -0,0 +1,53 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.server.webapp.dao;
|
||||
|
||||
/**
|
||||
* A remote log path for a log aggregation file controller.
|
||||
* <pre>
|
||||
* <ROOT_PATH>/%USER/<SUFFIX>
|
||||
* </pre>
|
||||
*/
|
||||
public class RemoteLogPathEntry {
|
||||
private String fileController;
|
||||
private String path;
|
||||
|
||||
//JAXB needs this
|
||||
public RemoteLogPathEntry() {}
|
||||
|
||||
public RemoteLogPathEntry(String fileController, String path) {
|
||||
this.fileController = fileController;
|
||||
this.path = path;
|
||||
}
|
||||
|
||||
public String getFileController() {
|
||||
return fileController;
|
||||
}
|
||||
|
||||
public void setFileController(String fileController) {
|
||||
this.fileController = fileController;
|
||||
}
|
||||
|
||||
public String getPath() {
|
||||
return path;
|
||||
}
|
||||
|
||||
public void setPath(String path) {
|
||||
this.path = path;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,50 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.server.webapp.dao;
|
||||
|
||||
import javax.xml.bind.annotation.XmlAccessType;
|
||||
import javax.xml.bind.annotation.XmlAccessorType;
|
||||
import javax.xml.bind.annotation.XmlElement;
|
||||
import javax.xml.bind.annotation.XmlRootElement;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Container of a list of {@link RemoteLogPathEntry}.
|
||||
*/
|
||||
@XmlRootElement(name = "remoteLogDirPathResult")
|
||||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
public class RemoteLogPaths {
|
||||
|
||||
@XmlElement(name = "paths")
|
||||
private List<RemoteLogPathEntry> paths;
|
||||
|
||||
//JAXB needs this
|
||||
public RemoteLogPaths() {}
|
||||
|
||||
public RemoteLogPaths(List<RemoteLogPathEntry> paths) {
|
||||
this.paths = paths;
|
||||
}
|
||||
|
||||
public List<RemoteLogPathEntry> getPaths() {
|
||||
return paths;
|
||||
}
|
||||
|
||||
public void setPaths(List<RemoteLogPathEntry> paths) {
|
||||
this.paths = paths;
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue