YARN-9728. Bugfix for escaping illegal xml characters for Resource Manager REST API.
Contributed by Prabhu Joseph
This commit is contained in:
parent
dc9abd27d9
commit
10144a580e
|
@ -3960,6 +3960,10 @@ public class YarnConfiguration extends Configuration {
|
|||
public static final boolean DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER =
|
||||
false;
|
||||
|
||||
public static final String FILTER_INVALID_XML_CHARS =
|
||||
"yarn.webapp.filter-invalid-xml-chars";
|
||||
public static final boolean DEFAULT_FILTER_INVALID_XML_CHARS = false;
|
||||
|
||||
// RM and NM CSRF props
|
||||
public static final String REST_CSRF = "webapp.rest-csrf.";
|
||||
public static final String RM_CSRF_PREFIX = RM_PREFIX + REST_CSRF;
|
||||
|
|
|
@ -3793,6 +3793,15 @@
|
|||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.webapp.filter-invalid-xml-chars</name>
|
||||
<value>false</value>
|
||||
<description>
|
||||
Flag to enable filter of invalid xml 1.0 characters present in the
|
||||
value of diagnostics field of apps output from RM WebService.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>
|
||||
The type of configuration store to use for scheduler configurations.
|
||||
|
|
|
@ -242,6 +242,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
|
|||
@VisibleForTesting
|
||||
boolean isCentralizedNodeLabelConfiguration = true;
|
||||
private boolean filterAppsByUser = false;
|
||||
private boolean filterInvalidXMLChars = false;
|
||||
|
||||
public final static String DELEGATION_TOKEN_HEADER =
|
||||
"Hadoop-YARN-RM-Delegation-Token";
|
||||
|
@ -257,6 +258,9 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
|
|||
this.filterAppsByUser = conf.getBoolean(
|
||||
YarnConfiguration.FILTER_ENTITY_LIST_BY_USER,
|
||||
YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
|
||||
this.filterInvalidXMLChars = conf.getBoolean(
|
||||
YarnConfiguration.FILTER_INVALID_XML_CHARS,
|
||||
YarnConfiguration.DEFAULT_FILTER_INVALID_XML_CHARS);
|
||||
}
|
||||
|
||||
RMWebServices(ResourceManager rm, Configuration conf,
|
||||
|
@ -551,6 +555,38 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
|
|||
return ni;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method ensures that the output String has only
|
||||
* valid XML unicode characters as specified by the
|
||||
* XML 1.0 standard. For reference, please see
|
||||
* <a href="http://www.w3.org/TR/2000/REC-xml-20001006#NT-Char">
|
||||
* the standard</a>.
|
||||
*
|
||||
* @param str The String whose invalid xml characters we want to escape.
|
||||
* @return The str String after escaping invalid xml characters.
|
||||
*/
|
||||
public static String escapeInvalidXMLCharacters(String str) {
|
||||
StringBuffer out = new StringBuffer();
|
||||
final int strlen = str.length();
|
||||
final String substitute = "\uFFFD";
|
||||
int idx = 0;
|
||||
while (idx < strlen) {
|
||||
final int cpt = str.codePointAt(idx);
|
||||
idx += Character.isSupplementaryCodePoint(cpt) ? 2 : 1;
|
||||
if ((cpt == 0x9) ||
|
||||
(cpt == 0xA) ||
|
||||
(cpt == 0xD) ||
|
||||
((cpt >= 0x20) && (cpt <= 0xD7FF)) ||
|
||||
((cpt >= 0xE000) && (cpt <= 0xFFFD)) ||
|
||||
((cpt >= 0x10000) && (cpt <= 0x10FFFF))) {
|
||||
out.append(Character.toChars(cpt));
|
||||
} else {
|
||||
out.append(substitute);
|
||||
}
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
@GET
|
||||
@Path(RMWSConsts.APPS)
|
||||
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
|
||||
|
@ -629,6 +665,17 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
|
|||
WebAppUtils.getHttpSchemePrefix(conf), deSelectFields);
|
||||
allApps.add(app);
|
||||
}
|
||||
|
||||
if (filterInvalidXMLChars) {
|
||||
final String format = hsr.getHeader(HttpHeaders.ACCEPT);
|
||||
if (format != null &&
|
||||
format.toLowerCase().contains(MediaType.APPLICATION_XML)) {
|
||||
for (AppInfo appInfo : allApps.getApps()) {
|
||||
appInfo.setNote(escapeInvalidXMLCharacters(appInfo.getNote()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return allApps;
|
||||
}
|
||||
|
||||
|
@ -985,8 +1032,18 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
|
|||
DeSelectFields deSelectFields = new DeSelectFields();
|
||||
deSelectFields.initFields(unselectedFields);
|
||||
|
||||
return new AppInfo(rm, app, hasAccess(app, hsr), hsr.getScheme() + "://",
|
||||
deSelectFields);
|
||||
AppInfo appInfo = new AppInfo(rm, app, hasAccess(app, hsr),
|
||||
hsr.getScheme() + "://", deSelectFields);
|
||||
|
||||
if (filterInvalidXMLChars) {
|
||||
final String format = hsr.getHeader(HttpHeaders.ACCEPT);
|
||||
if (format != null &&
|
||||
format.toLowerCase().contains(MediaType.APPLICATION_XML)) {
|
||||
appInfo.setNote(escapeInvalidXMLCharacters(appInfo.getNote()));
|
||||
}
|
||||
}
|
||||
|
||||
return appInfo;
|
||||
}
|
||||
|
||||
@GET
|
||||
|
|
|
@ -390,6 +390,10 @@ public class AppInfo {
|
|||
return this.diagnostics;
|
||||
}
|
||||
|
||||
public void setNote(String diagnosticsMsg) {
|
||||
this.diagnostics = diagnosticsMsg;
|
||||
}
|
||||
|
||||
public FinalApplicationStatus getFinalStatus() {
|
||||
return this.finalStatus;
|
||||
}
|
||||
|
|
|
@ -22,6 +22,7 @@ import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseS
|
|||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.isA;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
@ -30,11 +31,17 @@ import java.io.File;
|
|||
import java.io.StringReader;
|
||||
import java.security.Principal;
|
||||
import java.util.Arrays;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.HashMap;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import javax.ws.rs.core.HttpHeaders;
|
||||
import javax.ws.rs.core.MediaType;
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
|
@ -49,12 +56,21 @@ import org.apache.hadoop.util.VersionInfo;
|
|||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
||||
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
|
||||
import org.apache.hadoop.yarn.api.records.Priority;
|
||||
import org.apache.hadoop.yarn.api.records.QueueACL;
|
||||
import org.apache.hadoop.yarn.api.records.QueueState;
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.event.Dispatcher;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.*;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||
|
@ -870,6 +886,72 @@ public class TestRMWebServices extends JerseyTestBase {
|
|||
verifyClusterUserInfo(userInfo, "yarn", "admin");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInvalidXMLChars() throws Exception {
|
||||
ResourceManager mockRM = mock(ResourceManager.class);
|
||||
|
||||
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
|
||||
ApplicationReport appReport = ApplicationReport.newInstance(
|
||||
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
|
||||
"user", "queue", "appname", "host", 124, null,
|
||||
YarnApplicationState.FAILED, "java.lang.Exception: \u0001", "url",
|
||||
0, 0, 0, FinalApplicationStatus.FAILED, null, "N/A", 0.53789f, "YARN",
|
||||
null, null, false, Priority.newInstance(0), "high-mem", "high-mem");
|
||||
List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
|
||||
appReports.add(appReport);
|
||||
|
||||
GetApplicationsResponse response = mock(GetApplicationsResponse.class);
|
||||
when(response.getApplicationList()).thenReturn(appReports);
|
||||
ClientRMService clientRMService = mock(ClientRMService.class);
|
||||
when(clientRMService.getApplications(any(GetApplicationsRequest.class)))
|
||||
.thenReturn(response);
|
||||
when(mockRM.getClientRMService()).thenReturn(clientRMService);
|
||||
|
||||
RMContext rmContext = mock(RMContext.class);
|
||||
when(rmContext.getDispatcher()).thenReturn(mock(Dispatcher.class));
|
||||
|
||||
ApplicationSubmissionContext applicationSubmissionContext = mock(
|
||||
ApplicationSubmissionContext.class);
|
||||
when(applicationSubmissionContext.getUnmanagedAM()).thenReturn(true);
|
||||
|
||||
RMApp app = mock(RMApp.class);
|
||||
RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0),
|
||||
0, 0, new HashMap<>(), new HashMap<>());
|
||||
when(app.getDiagnostics()).thenReturn(
|
||||
new StringBuilder("java.lang.Exception: \u0001"));
|
||||
when(app.getApplicationId()).thenReturn(applicationId);
|
||||
when(app.getUser()).thenReturn("user");
|
||||
when(app.getName()).thenReturn("appname");
|
||||
when(app.getQueue()).thenReturn("queue");
|
||||
when(app.getRMAppMetrics()).thenReturn(appMetrics);
|
||||
when(app.getApplicationSubmissionContext()).thenReturn(
|
||||
applicationSubmissionContext);
|
||||
|
||||
ConcurrentMap<ApplicationId, RMApp> applications =
|
||||
new ConcurrentHashMap<>();
|
||||
applications.put(applicationId, app);
|
||||
|
||||
when(rmContext.getRMApps()).thenReturn(applications);
|
||||
when(mockRM.getRMContext()).thenReturn(rmContext);
|
||||
|
||||
Configuration conf = new YarnConfiguration();
|
||||
conf.setBoolean(YarnConfiguration.FILTER_INVALID_XML_CHARS, true);
|
||||
RMWebServices webSvc = new RMWebServices(mockRM, conf, mock(
|
||||
HttpServletResponse.class));
|
||||
|
||||
HttpServletRequest mockHsr = mock(HttpServletRequest.class);
|
||||
when(mockHsr.getHeader(HttpHeaders.ACCEPT)).
|
||||
thenReturn(MediaType.APPLICATION_XML);
|
||||
Set<String> emptySet = Collections.unmodifiableSet(Collections.emptySet());
|
||||
|
||||
AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
|
||||
null, null, null, null, null, null, null, emptySet, emptySet, null);
|
||||
|
||||
assertEquals("Incorrect Number of Apps", 1, appsInfo.getApps().size());
|
||||
assertEquals("Invalid XML Characters Present",
|
||||
"java.lang.Exception: \uFFFD", appsInfo.getApps().get(0).getNote());
|
||||
}
|
||||
|
||||
public void verifyClusterUserInfo(ClusterUserInfo userInfo,
|
||||
String rmLoginUser, String requestedUser) {
|
||||
assertEquals("rmLoginUser doesn't match: ",
|
||||
|
|
Loading…
Reference in New Issue