From f6cd1bcf896ac91157dd02b0ae988c06eb70706b Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Thu, 9 Jun 2016 12:30:58 -0700 Subject: [PATCH] =?UTF-8?q?YARN-5191.=20Renamed=20the=20newly=20added=20?= =?UTF-8?q?=E2=80=9Cdownload=3Dtrue=E2=80=9D=20option=20for=20getting=20lo?= =?UTF-8?q?gs=20via=20NMWebServices=20and=20AHSWebServices=20to=20be=20a?= =?UTF-8?q?=20better=20"format"=20option.=20(Xuan=20Gong=20via=20vinodkv)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 9378d9428f127eff7acd6c13544016cdbf2d65fb) --- .../hadoop/yarn/webapp/util/WebAppUtils.java | 18 ++++++++++ .../webapp/AHSWebServices.java | 36 +++++++++++-------- .../nodemanager/webapp/NMWebServices.java | 33 ++++++++++------- .../nodemanager/webapp/TestNMWebServices.java | 15 +++++++- 4 files changed, 74 insertions(+), 28 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index faf4a774447..3aa773aea6b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -24,6 +24,7 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -400,4 +401,21 @@ public class WebAppUtils { } return aid; } + + public static String getSupportedLogContentType(String format) { + if (format.equalsIgnoreCase("text")) { + return "text/plain"; + } else if (format.equalsIgnoreCase("octet-stream")) { + return "application/octet-stream"; + } + return null; + } + + public static String getDefaultLogContentType() { + return "text/plain"; + } + + public static List listSupportedLogContentType() { + return Arrays.asList("text", "octet-stream"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index 59dbd443c6e..692b1720c89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.base.Joiner; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -212,7 +213,7 @@ public class AHSWebServices extends WebServices { @Context HttpServletResponse res, @PathParam("containerid") String containerIdStr, @PathParam("filename") String filename, - @QueryParam("download") String download, + @QueryParam("format") String format, @QueryParam("size") String size) { init(res); ContainerId containerId; @@ -223,9 +224,6 @@ public class AHSWebServices extends WebServices { "Invalid ContainerId: " + containerIdStr); } - boolean downloadFile = parseBooleanParam(download); - - final long length = parseLongParam(size); ApplicationId appId = containerId.getApplicationAttemptId() @@ -236,7 +234,7 @@ public class AHSWebServices extends WebServices { } catch (Exception ex) { // directly find logs from HDFS. return sendStreamOutputResponse(appId, null, null, containerIdStr, - filename, downloadFile, length); + filename, format, length); } String appOwner = appInfo.getUser(); @@ -250,7 +248,7 @@ public class AHSWebServices extends WebServices { if (isFinishedState(appInfo.getAppState())) { // directly find logs from HDFS. return sendStreamOutputResponse(appId, appOwner, null, containerIdStr, - filename, downloadFile, length); + filename, format, length); } return createBadResponse(Status.INTERNAL_SERVER_ERROR, "Can not get ContainerInfo for the container: " + containerId); @@ -270,7 +268,7 @@ public class AHSWebServices extends WebServices { return response.build(); } else if (isFinishedState(appInfo.getAppState())) { return sendStreamOutputResponse(appId, appOwner, nodeId, - containerIdStr, filename, downloadFile, length); + containerIdStr, filename, format, length); } else { return createBadResponse(Status.NOT_FOUND, "The application is not at Running or Finished State."); @@ -293,13 +291,19 @@ public class AHSWebServices extends WebServices { return response; } - private boolean parseBooleanParam(String param) { - return ("true").equalsIgnoreCase(param); - } - private Response sendStreamOutputResponse(ApplicationId appId, String appOwner, String nodeId, String containerIdStr, - String fileName, boolean downloadFile, long bytes) { + String fileName, String format, long bytes) { + String contentType = WebAppUtils.getDefaultLogContentType(); + if (format != null && !format.isEmpty()) { + contentType = WebAppUtils.getSupportedLogContentType(format); + if (contentType == null) { + String errorMessage = "The valid values for the parameter : format " + + "are " + WebAppUtils.listSupportedLogContentType(); + return Response.status(Status.BAD_REQUEST).entity(errorMessage) + .build(); + } + } StreamingOutput stream = null; try { stream = getStreamingOutput(appId, appOwner, nodeId, @@ -313,9 +317,11 @@ public class AHSWebServices extends WebServices { "Can not get log for container: " + containerIdStr); } ResponseBuilder response = Response.ok(stream); - if (downloadFile) { - response.header("Content-Type", "application/octet-stream"); - } + response.header("Content-Type", contentType); + // Sending the X-Content-Type-Options response header with the value + // nosniff will prevent Internet Explorer from MIME-sniffing a response + // away from the declared content-type. + response.header("X-Content-Type-Options", "nosniff"); return response.build(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 943f3cc373e..efc0e7e6d96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -206,6 +206,10 @@ public class NMWebServices { * The container ID * @param filename * The name of the log file + * @param format + * The content type + * @param size + * the size of the log file * @return * The contents of the container's log file */ @@ -216,7 +220,7 @@ public class NMWebServices { @Unstable public Response getLogs(@PathParam("containerid") String containerIdStr, @PathParam("filename") String filename, - @QueryParam("download") String download, + @QueryParam("format") String format, @QueryParam("size") String size) { ContainerId containerId; try { @@ -234,8 +238,18 @@ public class NMWebServices { } catch (YarnException ex) { return Response.serverError().entity(ex.getMessage()).build(); } - boolean downloadFile = parseBooleanParam(download); final long bytes = parseLongParam(size); + String contentType = WebAppUtils.getDefaultLogContentType(); + if (format != null && !format.isEmpty()) { + contentType = WebAppUtils.getSupportedLogContentType(format); + if (contentType == null) { + String errorMessage = "The valid values for the parameter : format " + + "are " + WebAppUtils.listSupportedLogContentType(); + return Response.status(Status.BAD_REQUEST).entity(errorMessage) + .build(); + } + } + try { final FileInputStream fis = ContainerLogsUtils.openLogFileForRead( containerIdStr, logFile, nmContext); @@ -288,22 +302,17 @@ public class NMWebServices { } }; ResponseBuilder resp = Response.ok(stream); - if (downloadFile) { - resp.header("Content-Type", "application/octet-stream"); - } + resp.header("Content-Type", contentType); + // Sending the X-Content-Type-Options response header with the value + // nosniff will prevent Internet Explorer from MIME-sniffing a response + // away from the declared content-type. + resp.header("X-Content-Type-Options", "nosniff"); return resp.build(); } catch (IOException ex) { return Response.serverError().entity(ex.getMessage()).build(); } } - private boolean parseBooleanParam(String param) { - if (param != null) { - return ("true").equalsIgnoreCase(param); - } - return false; - } - private long parseLongParam(String bytes) { if (bytes == null || bytes.isEmpty()) { return Long.MAX_VALUE; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index 4e2feee0d3e..a4305da4563 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.junit.AfterClass; @@ -389,18 +390,30 @@ public class TestNMWebServices extends JerseyTestBase { .queryParam("size", "-10000") .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); responseText = response.getEntity(String.class); + assertEquals("text/plain", response.getType().toString()); assertEquals(fullTextSize, responseText.getBytes().length); assertEquals(logMessage, responseText); // ask and download it response = r.path("ws").path("v1").path("node").path("containerlogs") - .path(containerIdStr).path(filename).queryParam("download", "true") + .path(containerIdStr).path(filename) + .queryParam("format", "octet-stream") .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); responseText = response.getEntity(String.class); assertEquals(logMessage, responseText); assertEquals(200, response.getStatus()); assertEquals("application/octet-stream", response.getType().toString()); + // specify a invalid format value + response = r.path("ws").path("v1").path("node").path("containerlogs") + .path(containerIdStr).path(filename) + .queryParam("format", "123") + .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); + responseText = response.getEntity(String.class); + assertEquals("The valid values for the parameter : format are " + + WebAppUtils.listSupportedLogContentType(), responseText); + assertEquals(400, response.getStatus()); + // ask for file that doesn't exist response = r.path("ws").path("v1").path("node") .path("containerlogs").path(containerIdStr).path("uhhh")