Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-9814. JobHistoryServer can't delete aggregated files, if remote app root directory is created by NodeManager. Contributed by Adam Antal.

This commit is contained in:
Sunil G 2019-09-18 09:27:41 +05:30
parent f8c14326ee
commit 01d7924473
4 changed files with 122 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -1479,6 +1479,12 @@ public static boolean isAclEnabled(Configuration conf) {
public static final boolean DEFAULT_NM_REMOTE_APP_LOG_DIR_INCLUDE_OLDER =
true;
/**
* Specifies the group of the aggregated log directory.
*/
public static final String NM_REMOTE_APP_LOG_DIR_GROUPNAME =
NM_PREFIX + "remote-app-log-dir.groupname";
public static final String YARN_LOG_SERVER_URL =
YARN_PREFIX + "log.server.url";

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
View File

@ -52,7 +52,6 @@
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
import org.apache.hadoop.yarn.webapp.View.ViewContext;
@ -346,13 +345,23 @@ public void verifyAndCreateRemoteLogDir() {
}
UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
String primaryGroupName = null;
try {
primaryGroupName = loginUser.getPrimaryGroupName();
} catch (IOException e) {
LOG.warn("No primary group found. The remote root log directory" +
" will be created with the HDFS superuser being its group " +
"owner. JobHistoryServer may be unable to read the directory.");
String primaryGroupName = conf.get(
YarnConfiguration.NM_REMOTE_APP_LOG_DIR_GROUPNAME);
if (primaryGroupName == null || primaryGroupName.isEmpty()) {
try {
primaryGroupName = loginUser.getPrimaryGroupName();
} catch (IOException e) {
LOG.warn("No primary group found. The remote root log directory" +
" will be created with the HDFS superuser being its " +
"group owner. JobHistoryServer may be unable to read " +
"the directory.");
}
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("The group of remote root log directory has been " +
"determined by the configuration and set to " +
primaryGroupName);
}
}
// set owner on the remote directory only if the primary group exists
if (primaryGroupName != null) {

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -1373,6 +1373,14 @@
<value>true</value>
</property>
<property>
<description>If the NodeManager creates the remote-app-log-dir folder,
it will be created with this groupname.
</description>
<name>yarn.nodemanager.remote-app-log-dir.groupname</name>
<value></value>
</property>
<property>
<description>Generate additional logs about container launches.
Currently, this creates a copy of the launch script and lists the

91
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java Normal file
View File

@ -0,0 +1,91 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.logaggregation.filecontroller;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.Test;
import org.mockito.Mockito;
import java.io.FileNotFoundException;
import java.net.URI;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
/**
* Test for the abstract {@link LogAggregationFileController} class,
* checking its core functionality.
*/
public class TestLogAggregationFileController {
@Test
public void testRemoteDirCreationDefault() throws Exception {
FileSystem fs = mock(FileSystem.class);
doReturn(new URI("")).when(fs).getUri();
doThrow(FileNotFoundException.class).when(fs)
.getFileStatus(any(Path.class));
Configuration conf = new Configuration();
LogAggregationFileController controller = mock(
LogAggregationFileController.class, Mockito.CALLS_REAL_METHODS);
doReturn(fs).when(controller).getFileSystem(any(Configuration.class));
UserGroupInformation ugi = UserGroupInformation.createUserForTesting(
"yarn_user", new String[] {"yarn_group", "other_group"});
UserGroupInformation.setLoginUser(ugi);
controller.initialize(conf, "TFile");
controller.verifyAndCreateRemoteLogDir();
verify(fs).setOwner(any(), eq("yarn_user"), eq("yarn_group"));
}
@Test
public void testRemoteDirCreationWithCustomGroup() throws Exception {
String testGroupName = "testGroup";
FileSystem fs = mock(FileSystem.class);
doReturn(new URI("")).when(fs).getUri();
doThrow(FileNotFoundException.class).when(fs)
.getFileStatus(any(Path.class));
Configuration conf = new Configuration();
conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_GROUPNAME, testGroupName);
LogAggregationFileController controller = mock(
LogAggregationFileController.class, Mockito.CALLS_REAL_METHODS);
doReturn(fs).when(controller).getFileSystem(any(Configuration.class));
UserGroupInformation ugi = UserGroupInformation.createUserForTesting(
"yarn_user", new String[] {"yarn_group", "other_group"});
UserGroupInformation.setLoginUser(ugi);
controller.initialize(conf, "TFile");
controller.verifyAndCreateRemoteLogDir();
verify(fs).setOwner(any(), eq("yarn_user"), eq(testGroupName));
}
}