diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 9c62827cb17..7b0590522bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1479,6 +1479,12 @@ public static boolean isAclEnabled(Configuration conf) { public static final boolean DEFAULT_NM_REMOTE_APP_LOG_DIR_INCLUDE_OLDER = true; + /** + * Specifies the group of the aggregated log directory. + */ + public static final String NM_REMOTE_APP_LOG_DIR_GROUPNAME = + NM_PREFIX + "remote-app-log-dir.groupname"; + public static final String YARN_LOG_SERVER_URL = YARN_PREFIX + "log.server.url"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java index 661e3219d33..001f4f53e7f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java @@ -52,7 +52,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils; import org.apache.hadoop.yarn.webapp.View.ViewContext; @@ -346,13 +345,23 @@ public void verifyAndCreateRemoteLogDir() { } UserGroupInformation loginUser = UserGroupInformation.getLoginUser(); - String primaryGroupName = null; - try { - primaryGroupName = loginUser.getPrimaryGroupName(); - } catch (IOException e) { - LOG.warn("No primary group found. The remote root log directory" + - " will be created with the HDFS superuser being its group " + - "owner. JobHistoryServer may be unable to read the directory."); + String primaryGroupName = conf.get( + YarnConfiguration.NM_REMOTE_APP_LOG_DIR_GROUPNAME); + if (primaryGroupName == null || primaryGroupName.isEmpty()) { + try { + primaryGroupName = loginUser.getPrimaryGroupName(); + } catch (IOException e) { + LOG.warn("No primary group found. The remote root log directory" + + " will be created with the HDFS superuser being its " + + "group owner. JobHistoryServer may be unable to read " + + "the directory."); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("The group of remote root log directory has been " + + "determined by the configuration and set to " + + primaryGroupName); + } } // set owner on the remote directory only if the primary group exists if (primaryGroupName != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index b856536494c..55e908de4ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1373,6 +1373,14 @@ true + + If the NodeManager creates the remote-app-log-dir folder, + it will be created with this groupname. + + yarn.nodemanager.remote-app-log-dir.groupname + + + Generate additional logs about container launches. Currently, this creates a copy of the launch script and lists the diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java new file mode 100644 index 00000000000..5ade0faabd8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/TestLogAggregationFileController.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.logaggregation.filecontroller; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.FileNotFoundException; +import java.net.URI; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +/** + * Test for the abstract {@link LogAggregationFileController} class, + * checking its core functionality. + */ +public class TestLogAggregationFileController { + + @Test + public void testRemoteDirCreationDefault() throws Exception { + FileSystem fs = mock(FileSystem.class); + doReturn(new URI("")).when(fs).getUri(); + doThrow(FileNotFoundException.class).when(fs) + .getFileStatus(any(Path.class)); + + Configuration conf = new Configuration(); + LogAggregationFileController controller = mock( + LogAggregationFileController.class, Mockito.CALLS_REAL_METHODS); + doReturn(fs).when(controller).getFileSystem(any(Configuration.class)); + + UserGroupInformation ugi = UserGroupInformation.createUserForTesting( + "yarn_user", new String[] {"yarn_group", "other_group"}); + UserGroupInformation.setLoginUser(ugi); + + controller.initialize(conf, "TFile"); + controller.verifyAndCreateRemoteLogDir(); + + verify(fs).setOwner(any(), eq("yarn_user"), eq("yarn_group")); + } + + @Test + public void testRemoteDirCreationWithCustomGroup() throws Exception { + String testGroupName = "testGroup"; + + FileSystem fs = mock(FileSystem.class); + doReturn(new URI("")).when(fs).getUri(); + doThrow(FileNotFoundException.class).when(fs) + .getFileStatus(any(Path.class)); + + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_GROUPNAME, testGroupName); + LogAggregationFileController controller = mock( + LogAggregationFileController.class, Mockito.CALLS_REAL_METHODS); + doReturn(fs).when(controller).getFileSystem(any(Configuration.class)); + + UserGroupInformation ugi = UserGroupInformation.createUserForTesting( + "yarn_user", new String[] {"yarn_group", "other_group"}); + UserGroupInformation.setLoginUser(ugi); + + controller.initialize(conf, "TFile"); + controller.verifyAndCreateRemoteLogDir(); + + verify(fs).setOwner(any(), eq("yarn_user"), eq(testGroupName)); + } +}