YARN-8319. More YARN pages need to honor yarn.resourcemanager.display.per-user-apps. Contributed by Sunil G.
This commit is contained in:
parent
4cc0c9b0ba
commit
c05b5d424b
|
@ -121,6 +121,10 @@ public class YarnConfiguration extends Configuration {
|
|||
new DeprecationDelta(RM_ZK_RETRY_INTERVAL_MS,
|
||||
CommonConfigurationKeys.ZK_RETRY_INTERVAL_MS),
|
||||
});
|
||||
Configuration.addDeprecations(new DeprecationDelta[] {
|
||||
new DeprecationDelta("yarn.resourcemanager.display.per-user-apps",
|
||||
FILTER_ENTITY_LIST_BY_USER)
|
||||
});
|
||||
}
|
||||
|
||||
//Configurations
|
||||
|
@ -3569,11 +3573,16 @@ public class YarnConfiguration extends Configuration {
|
|||
public static final String NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_SCRIPT_OPTS =
|
||||
NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PREFIX + "opts";
|
||||
|
||||
/*
|
||||
/**
|
||||
* Support to view apps for given user in secure cluster.
|
||||
* @deprecated This field is deprecated for {@link #FILTER_ENTITY_LIST_BY_USER}
|
||||
*/
|
||||
@Deprecated
|
||||
public static final String DISPLAY_APPS_FOR_LOGGED_IN_USER =
|
||||
RM_PREFIX + "display.per-user-apps";
|
||||
|
||||
public static final String FILTER_ENTITY_LIST_BY_USER =
|
||||
"yarn.webapp.filter-entity-list-by-user";
|
||||
public static final boolean DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER =
|
||||
false;
|
||||
|
||||
|
|
|
@ -182,6 +182,8 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
|
|||
// Ignore deprecated properties
|
||||
configurationPrefixToSkipCompare
|
||||
.add(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS);
|
||||
configurationPrefixToSkipCompare
|
||||
.add(YarnConfiguration.DISPLAY_APPS_FOR_LOGGED_IN_USER);
|
||||
|
||||
// Allocate for usage
|
||||
xmlPropsToSkipCompare = new HashSet<String>();
|
||||
|
|
|
@ -3529,7 +3529,7 @@
|
|||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.resourcemanager.display.per-user-apps</name>
|
||||
<name>yarn.webapp.filter-entity-list-by-user</name>
|
||||
<value>false</value>
|
||||
<description>
|
||||
Flag to enable display of applications per user as an admin
|
||||
|
|
|
@ -22,6 +22,7 @@ import java.io.FileInputStream;
|
|||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.charset.Charset;
|
||||
import java.security.Principal;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
|
@ -54,6 +55,8 @@ import org.apache.commons.lang.StringUtils;
|
|||
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||
import org.apache.hadoop.http.JettyUtils;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
|
@ -99,6 +102,7 @@ public class NMWebServices {
|
|||
.getRecordFactory(null);
|
||||
private final String redirectWSUrl;
|
||||
private final LogAggregationFileControllerFactory factory;
|
||||
private boolean filterAppsByUser = false;
|
||||
|
||||
private @javax.ws.rs.core.Context
|
||||
HttpServletRequest request;
|
||||
|
@ -119,6 +123,15 @@ public class NMWebServices {
|
|||
YarnConfiguration.YARN_LOG_SERVER_WEBSERVICE_URL);
|
||||
this.factory = new LogAggregationFileControllerFactory(
|
||||
this.nmContext.getConf());
|
||||
this.filterAppsByUser = this.nmContext.getConf().getBoolean(
|
||||
YarnConfiguration.FILTER_ENTITY_LIST_BY_USER,
|
||||
YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
|
||||
}
|
||||
|
||||
public NMWebServices(final Context nm, final ResourceView view,
|
||||
final WebApp webapp, HttpServletResponse response) {
|
||||
this(nm, view, webapp);
|
||||
this.response = response;
|
||||
}
|
||||
|
||||
private void init() {
|
||||
|
@ -146,7 +159,8 @@ public class NMWebServices {
|
|||
@Path("/apps")
|
||||
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
|
||||
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
|
||||
public AppsInfo getNodeApps(@QueryParam("state") String stateQuery,
|
||||
public AppsInfo getNodeApps(@javax.ws.rs.core.Context HttpServletRequest hsr,
|
||||
@QueryParam("state") String stateQuery,
|
||||
@QueryParam("user") String userQuery) {
|
||||
init();
|
||||
AppsInfo allApps = new AppsInfo();
|
||||
|
@ -169,6 +183,14 @@ public class NMWebServices {
|
|||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Allow only application-owner/admin for any type of access on the
|
||||
// application.
|
||||
if (filterAppsByUser
|
||||
&& !hasAccess(appInfo.getUser(), entry.getKey(), hsr)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
allApps.add(appInfo);
|
||||
}
|
||||
return allApps;
|
||||
|
@ -205,6 +227,16 @@ public class NMWebServices {
|
|||
}
|
||||
ContainerInfo info = new ContainerInfo(this.nmContext, entry.getValue(),
|
||||
uriInfo.getBaseUri().toString(), webapp.name(), hsr.getRemoteUser());
|
||||
|
||||
ApplicationId appId = entry.getKey().getApplicationAttemptId()
|
||||
.getApplicationId();
|
||||
// Allow only application-owner/admin for any type of access on the
|
||||
// application.
|
||||
if (filterAppsByUser
|
||||
&& !hasAccess(entry.getValue().getUser(), appId, hsr)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
allContainers.add(info);
|
||||
}
|
||||
return allContainers;
|
||||
|
@ -553,4 +585,33 @@ public class NMWebServices {
|
|||
res.header("Location", redirectPath.toString());
|
||||
return res.build();
|
||||
}
|
||||
|
||||
protected Boolean hasAccess(String user, ApplicationId appId,
|
||||
HttpServletRequest hsr) {
|
||||
// Check for the authorization.
|
||||
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
|
||||
|
||||
if (callerUGI != null && !(this.nmContext.getApplicationACLsManager()
|
||||
.checkAccess(callerUGI, ApplicationAccessType.VIEW_APP, user, appId))) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private UserGroupInformation getCallerUserGroupInformation(
|
||||
HttpServletRequest hsr, boolean usePrincipal) {
|
||||
|
||||
String remoteUser = hsr.getRemoteUser();
|
||||
if (usePrincipal) {
|
||||
Principal princ = hsr.getUserPrincipal();
|
||||
remoteUser = princ == null ? null : princ.getName();
|
||||
}
|
||||
|
||||
UserGroupInformation callerUGI = null;
|
||||
if (remoteUser != null) {
|
||||
callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
|
||||
}
|
||||
|
||||
return callerUGI;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -22,12 +22,17 @@ import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseS
|
|||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.security.Principal;
|
||||
import java.util.HashMap;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import javax.ws.rs.core.MediaType;
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
|
@ -44,11 +49,13 @@ import org.apache.hadoop.yarn.server.nodemanager.Context;
|
|||
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp;
|
||||
import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.AppsInfo;
|
||||
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
|
||||
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
||||
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
|
||||
|
@ -192,25 +199,28 @@ public class TestNMWebServicesApps extends JerseyTestBase {
|
|||
|
||||
private HashMap<String, String> addAppContainers(Application app)
|
||||
throws IOException {
|
||||
return addAppContainers(app, nmContext);
|
||||
}
|
||||
|
||||
private HashMap<String, String> addAppContainers(Application app,
|
||||
Context context) throws IOException {
|
||||
Dispatcher dispatcher = new AsyncDispatcher();
|
||||
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
|
||||
app.getAppId(), 1);
|
||||
ApplicationAttemptId appAttemptId = BuilderUtils
|
||||
.newApplicationAttemptId(app.getAppId(), 1);
|
||||
Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
|
||||
app.getUser(), app.getAppId(), 1);
|
||||
Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
|
||||
app.getUser(), app.getAppId(), 2);
|
||||
nmContext.getContainers()
|
||||
.put(container1.getContainerId(), container1);
|
||||
nmContext.getContainers()
|
||||
.put(container2.getContainerId(), container2);
|
||||
context.getContainers().put(container1.getContainerId(), container1);
|
||||
context.getContainers().put(container2.getContainerId(), container2);
|
||||
|
||||
app.getContainers().put(container1.getContainerId(), container1);
|
||||
app.getContainers().put(container2.getContainerId(), container2);
|
||||
HashMap<String, String> hash = new HashMap<String, String>();
|
||||
hash.put(container1.getContainerId().toString(), container1
|
||||
.getContainerId().toString());
|
||||
hash.put(container2.getContainerId().toString(), container2
|
||||
.getContainerId().toString());
|
||||
hash.put(container1.getContainerId().toString(),
|
||||
container1.getContainerId().toString());
|
||||
hash.put(container2.getContainerId().toString(),
|
||||
container2.getContainerId().toString());
|
||||
return hash;
|
||||
}
|
||||
|
||||
|
@ -721,4 +731,42 @@ public class TestNMWebServicesApps extends JerseyTestBase {
|
|||
user);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNodeAppsUserFiltering() throws JSONException, Exception {
|
||||
Configuration yarnConf = new Configuration();
|
||||
yarnConf.setBoolean(YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, true);
|
||||
yarnConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
||||
yarnConf.setStrings(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
||||
ApplicationACLsManager aclManager = new ApplicationACLsManager(yarnConf);
|
||||
|
||||
NMContext context = new NodeManager.NMContext(null, null, dirsHandler,
|
||||
aclManager, null, false, yarnConf);
|
||||
Application app = new MockApp(1);
|
||||
context.getApplications().put(app.getAppId(), app);
|
||||
addAppContainers(app, context);
|
||||
Application app2 = new MockApp("foo", 1234, 2);
|
||||
context.getApplications().put(app2.getAppId(), app2);
|
||||
addAppContainers(app2, context);
|
||||
|
||||
// User "foo" could only see its own apps/containers.
|
||||
NMWebServices webSvc = new NMWebServices(context, null, nmWebApp,
|
||||
mock(HttpServletResponse.class));
|
||||
HttpServletRequest mockHsr = mockHttpServletRequestByUserName("foo");
|
||||
AppsInfo appsInfo = webSvc.getNodeApps(mockHsr, null, null);
|
||||
assertEquals(1, appsInfo.getApps().size());
|
||||
|
||||
// Admin could see all apps and containers.
|
||||
HttpServletRequest mockHsrAdmin = mockHttpServletRequestByUserName("admin");
|
||||
AppsInfo appsInfo2 = webSvc.getNodeApps(mockHsrAdmin, null, null);
|
||||
assertEquals(2, appsInfo2.getApps().size());
|
||||
}
|
||||
|
||||
private HttpServletRequest mockHttpServletRequestByUserName(String username) {
|
||||
HttpServletRequest mockHsr = mock(HttpServletRequest.class);
|
||||
when(mockHsr.getRemoteUser()).thenReturn(username);
|
||||
Principal principal = mock(Principal.class);
|
||||
when(principal.getName()).thenReturn(username);
|
||||
when(mockHsr.getUserPrincipal()).thenReturn(principal);
|
||||
return mockHsr;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -216,7 +216,7 @@ public class ClientRMService extends AbstractService implements
|
|||
private ReservationSystem reservationSystem;
|
||||
private ReservationInputValidator rValidator;
|
||||
|
||||
private boolean displayPerUserApps = false;
|
||||
private boolean filterAppsByUser = false;
|
||||
|
||||
private static final EnumSet<RMAppState> ACTIVE_APP_STATES = EnumSet.of(
|
||||
RMAppState.ACCEPTED, RMAppState.RUNNING);
|
||||
|
@ -283,8 +283,8 @@ public class ClientRMService extends AbstractService implements
|
|||
refreshServiceAcls(conf, RMPolicyProvider.getInstance());
|
||||
}
|
||||
|
||||
this.displayPerUserApps = conf.getBoolean(
|
||||
YarnConfiguration.DISPLAY_APPS_FOR_LOGGED_IN_USER,
|
||||
this.filterAppsByUser = conf.getBoolean(
|
||||
YarnConfiguration.FILTER_ENTITY_LIST_BY_USER,
|
||||
YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
|
||||
|
||||
this.server.start();
|
||||
|
@ -922,7 +922,7 @@ public class ClientRMService extends AbstractService implements
|
|||
|
||||
// Given RM is configured to display apps per user, skip apps to which
|
||||
// this caller doesn't have access to view.
|
||||
if (displayPerUserApps && !allowAccess) {
|
||||
if (filterAppsByUser && !allowAccess) {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -1840,6 +1840,6 @@ public class ClientRMService extends AbstractService implements
|
|||
|
||||
@VisibleForTesting
|
||||
public void setDisplayPerUserApps(boolean displayPerUserApps) {
|
||||
this.displayPerUserApps = displayPerUserApps;
|
||||
this.filterAppsByUser = displayPerUserApps;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -228,7 +228,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
|
|||
|
||||
@VisibleForTesting
|
||||
boolean isCentralizedNodeLabelConfiguration = true;
|
||||
private boolean displayPerUserApps = false;
|
||||
private boolean filterAppsByUser = false;
|
||||
|
||||
public final static String DELEGATION_TOKEN_HEADER =
|
||||
"Hadoop-YARN-RM-Delegation-Token";
|
||||
|
@ -241,8 +241,8 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
|
|||
this.conf = conf;
|
||||
isCentralizedNodeLabelConfiguration =
|
||||
YarnConfiguration.isCentralizedNodeLabelConfiguration(conf);
|
||||
this.displayPerUserApps = conf.getBoolean(
|
||||
YarnConfiguration.DISPLAY_APPS_FOR_LOGGED_IN_USER,
|
||||
this.filterAppsByUser = conf.getBoolean(
|
||||
YarnConfiguration.FILTER_ENTITY_LIST_BY_USER,
|
||||
YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
|
||||
}
|
||||
|
||||
|
@ -654,7 +654,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
|
|||
boolean allowAccess = hasAccess(rmapp, hsr);
|
||||
// Given RM is configured to display apps per user, skip apps to which
|
||||
// this caller doesn't have access to view.
|
||||
if (displayPerUserApps && !allowAccess) {
|
||||
if (filterAppsByUser && !allowAccess) {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
|
@ -23,6 +23,7 @@ import java.text.ParseException;
|
|||
import java.text.SimpleDateFormat;
|
||||
import java.util.Collections;
|
||||
import java.util.Date;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.TimeZone;
|
||||
|
@ -42,12 +43,15 @@ import javax.ws.rs.core.Response;
|
|||
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.http.JettyUtils;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.util.Time;
|
||||
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
|
||||
import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
|
||||
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
|
||||
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
|
||||
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
|
||||
import org.apache.hadoop.yarn.webapp.BadRequestException;
|
||||
|
@ -1450,6 +1454,19 @@ public class TimelineReaderWebServices {
|
|||
long endTime = Time.monotonicNow();
|
||||
if (entities == null) {
|
||||
entities = Collections.emptySet();
|
||||
} else if (isDisplayEntityPerUserFilterEnabled(
|
||||
timelineReaderManager.getConfig())) {
|
||||
Set<TimelineEntity> userEntities = new LinkedHashSet<>();
|
||||
userEntities.addAll(entities);
|
||||
for (TimelineEntity entity : userEntities) {
|
||||
if (entity.getInfo() != null) {
|
||||
String userId =
|
||||
(String) entity.getInfo().get(FlowActivityEntity.USER_INFO_KEY);
|
||||
if (!validateAuthUserWithEntityUser(callerUGI, userId)) {
|
||||
entities.remove(entity);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
LOG.info("Processed URL " + url +
|
||||
" (Took " + (endTime - startTime) + " ms.)");
|
||||
|
@ -3403,4 +3420,20 @@ public class TimelineReaderWebServices {
|
|||
"Processed URL " + url + " (Took " + (endTime - startTime) + " ms.)");
|
||||
return entities;
|
||||
}
|
||||
|
||||
private boolean isDisplayEntityPerUserFilterEnabled(Configuration config) {
|
||||
return config
|
||||
.getBoolean(YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, false);
|
||||
}
|
||||
|
||||
private boolean validateAuthUserWithEntityUser(UserGroupInformation ugi,
|
||||
String entityUser) {
|
||||
String authUser = TimelineReaderWebServicesUtils.getUserName(ugi);
|
||||
String requestedUser = TimelineReaderWebServicesUtils.parseStr(entityUser);
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug(
|
||||
"Authenticated User: " + authUser + " Requested User:" + entityUser);
|
||||
}
|
||||
return authUser.equals(requestedUser);
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue